GNU Octave  3.8.0
A high-level interpreted language, primarily intended for numerical computations, mostly compatible with Matlab
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
url-transfer.cc
Go to the documentation of this file.
1 /*
2 
3 Copyright (C) 2013 John W. Eaton
4 Copyright (C) 2006-2013 Alexander Barth
5 Copyright (C) 2009 David Bateman
6 
7 This file is part of Octave.
8 
9 Octave is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3 of the License, or (at your
12 option) any later version.
13 
14 Octave is distributed in the hope that it will be useful, but WITHOUT
15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 for more details.
18 
19 You should have received a copy of the GNU General Public License
20 along with Octave; see the file COPYING. If not, see
21 <http://www.gnu.org/licenses/>.
22 
23 */
24 
25 // Author: Alexander Barth <[email protected]>
26 // Author: jwe
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35 
36 #include "dir-ops.h"
37 #include "file-ops.h"
38 #include "file-stat.h"
39 #include "unwind-prot.h"
40 #include "url-transfer.h"
41 
42 #ifdef HAVE_CURL
43 #include <curl/curl.h>
44 #include <curl/curlver.h>
45 #include <curl/easy.h>
46 #endif
47 
48 void
49 base_url_transfer::delete_file (const std::string& file)
50 {
51  octave_unlink (file);
52 }
53 
54 void
55 base_url_transfer::mget_directory (const std::string& directory,
56  const std::string& target)
57 {
58  std::string sep = file_ops::dir_sep_str ();
59  file_stat fs (directory);
60 
61  if (!fs || !fs.is_dir ())
62  {
63  std::string msg;
64  int status = octave_mkdir (directory, 0777, msg);
65 
66  if (status < 0)
67  {
68  ok = false;
69  errmsg = "__ftp_mget__: can not create directory '"
70  + target + sep + directory + "': " + msg;
71  return;
72  }
73  }
74 
75  cwd (directory);
76 
77  if (good ())
78  {
79  unwind_protect_safe frame;
80 
81  frame.add_fcn (reset_path, this);
82 
83  string_vector sv = list ();
84 
85  for (octave_idx_type i = 0; i < sv.length (); i++)
86  {
87  time_t ftime;
88  bool fisdir;
89  double fsize;
90 
91  get_fileinfo (sv(i), fsize, ftime, fisdir);
92 
93  if (fisdir)
94  mget_directory (sv(i), target + directory + sep);
95  else
96  {
97  std::string realfile = target + directory + sep + sv(i);
98 
99  std::ofstream ofile (realfile.c_str (),
100  std::ios::out | std::ios::binary);
101 
102  if (! ofile.is_open ())
103  {
104  ok = false;
105  errmsg = "__ftp_mget__: unable to open file";
106  break;
107  }
108 
109  unwind_protect_safe frame2;
110 
111  frame2.add_fcn (delete_file, realfile);
112 
113  get (sv(i), ofile);
114 
115  ofile.close ();
116 
117  if (good ())
118  frame2.discard ();
119  }
120 
121  if (! good ())
122  break;
123  }
124  }
125 }
126 
128 base_url_transfer::mput_directory (const std::string& base,
129  const std::string& directory)
130 {
131  string_vector file_list;
132 
133  std::string realdir
134  = (base.length () == 0
135  ? directory : base + file_ops::dir_sep_str () + directory);
136 
137  mkdir (directory);
138 
139  if (! good ())
140  return file_list;
141 
142  cwd (directory);
143 
144  if (good ())
145  {
146  unwind_protect_safe frame;
147 
148  frame.add_fcn (reset_path, this);
149 
150  dir_entry dirlist (realdir);
151 
152  if (dirlist)
153  {
154  string_vector files = dirlist.read ();
155 
156  for (octave_idx_type i = 0; i < files.length (); i++)
157  {
158  std::string file = files (i);
159 
160  if (file == "." || file == "..")
161  continue;
162 
163  std::string realfile = realdir + file_ops::dir_sep_str () + file;
164  file_stat fs (realfile);
165 
166  if (! fs.exists ())
167  {
168  ok = false;
169  errmsg = "__ftp__mput: file '" + realfile
170  + "' does not exist";
171  break;
172  }
173 
174  if (fs.is_dir ())
175  {
176  file_list.append (mput_directory (realdir, file));
177 
178  if (! good ())
179  break;
180  }
181  else
182  {
183  // FIXME: Does ascii mode need to be flagged here?
184  std::ifstream ifile (realfile.c_str (), std::ios::in |
186 
187  if (! ifile.is_open ())
188  {
189  ok = false;
190  errmsg = "__ftp_mput__: unable to open file '"
191  + realfile + "'";
192  break;
193  }
194 
195  put (file, ifile);
196 
197  ifile.close ();
198 
199  if (! good ())
200  break;
201 
202  file_list.append (realfile);
203  }
204  }
205  }
206  else
207  {
208  ok = false;
209  errmsg = "__ftp_mput__: can not read the directory '"
210  + realdir + "'";
211  }
212  }
213 
214  return file_list;
215 }
216 
217 #if defined (HAVE_CURL)
218 
219 static int
220 write_data (void *buffer, size_t size, size_t nmemb, void *streamp)
221 {
222  std::ostream& stream = *(static_cast<std::ostream*> (streamp));
223  stream.write (static_cast<const char*> (buffer), size*nmemb);
224  return (stream.fail () ? 0 : size * nmemb);
225 }
226 
227 static int
228 read_data (void *buffer, size_t size, size_t nmemb, void *streamp)
229 {
230  std::istream& stream = *(static_cast<std::istream*> (streamp));
231  stream.read (static_cast<char*> (buffer), size*nmemb);
232  if (stream.eof ())
233  return stream.gcount ();
234  else
235  return (stream.fail () ? 0 : size * nmemb);
236 }
237 
238 static size_t
239 throw_away (void *, size_t size, size_t nmemb, void *)
240 {
241  return static_cast<size_t>(size * nmemb);
242 }
243 
244 // I'd love to rewrite this as a private method of the url_transfer
245 // class, but you can't pass the va_list from the wrapper SETOPT to
246 // the curl_easy_setopt function.
247 #define SETOPT(option, parameter) \
248  do \
249  { \
250  CURLcode res = curl_easy_setopt (curl, option, parameter); \
251  if (res != CURLE_OK) \
252  { \
253  ok = false; \
254  errmsg = curl_easy_strerror (res); \
255  return; \
256  } \
257  } \
258  while (0)
259 
260 // Same as above but with a return value.
261 #define SETOPTR(option, parameter) \
262  do \
263  { \
264  CURLcode res = curl_easy_setopt (curl, option, parameter); \
265  if (res != CURLE_OK) \
266  { \
267  ok = false; \
268  errmsg = curl_easy_strerror (res); \
269  return retval; \
270  } \
271  } \
272  while (0)
273 
275 {
276 public:
277 
279  : base_url_transfer (), curl (curl_easy_init ()), errnum (), url (),
280  userpwd ()
281  {
282  if (curl)
283  valid = true;
284  else
285  errmsg = "can not create curl object";
286  }
287 
288  curl_transfer (const std::string& host, const std::string& user_arg,
289  const std::string& passwd, std::ostream& os)
290  : base_url_transfer (host, user_arg, passwd, os),
291  curl (curl_easy_init ()), errnum (), url (), userpwd ()
292  {
293  if (curl)
294  valid = true;
295  else
296  {
297  errmsg = "can not create curl object";
298  return;
299  }
300 
301  init (user_arg, passwd, std::cin, os);
302 
303  url = "ftp://" + host;
304  SETOPT (CURLOPT_URL, url.c_str ());
305 
306  // Set up the link, with no transfer.
307  perform ();
308  }
309 
310  curl_transfer (const std::string& url_str, std::ostream& os)
311  : base_url_transfer (url_str, os), curl (curl_easy_init ()), errnum (),
312  url (), userpwd ()
313  {
314  if (curl)
315  valid = true;
316  else
317  {
318  errmsg = "can not create curl object";
319  return;
320  }
321 
322  init ("", "", std::cin, os);
323 
324  SETOPT (CURLOPT_NOBODY, 0);
325 
326  // Restore the default HTTP request method to GET after setting
327  // NOBODY to true (in the init method) and back to false (above).
328  // This is needed for backward compatibility with versions of
329  // libcurl < 7.18.2.
330  SETOPT (CURLOPT_HTTPGET, 1);
331  }
332 
334  {
335  if (curl)
336  curl_easy_cleanup (curl);
337  }
338 
339  void perform (void)
340  {
342 
343  errnum = curl_easy_perform (curl);
344 
345  if (errnum != CURLE_OK)
346  {
347  ok = false;
348  errmsg = curl_easy_strerror (errnum);
349  }
350 
352  }
353 
354  std::string lasterror (void) const
355  {
356  return std::string (curl_easy_strerror (errnum));
357  }
358 
359  std::ostream& set_ostream (std::ostream& os)
360  {
361  std::ostream& retval = *curr_ostream;
362  curr_ostream = &os;
363  SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (curr_ostream));
364  return retval;
365  }
366 
367  std::istream& set_istream (std::istream& is)
368  {
369  std::istream& retval = *curr_istream;
370  curr_istream = &is;
371  SETOPTR (CURLOPT_READDATA, static_cast<void*> (curr_istream));
372  return retval;
373  }
374 
375  void ascii (void)
376  {
377  ascii_mode = true;
378  SETOPT (CURLOPT_TRANSFERTEXT, 1);
379  }
380 
381  void binary (void)
382  {
383  ascii_mode = false;
384  SETOPT (CURLOPT_TRANSFERTEXT, 0);
385  }
386 
387  void cwd (const std::string& path)
388  {
389  ftp_file_or_dir_action (path, "cwd");
390  }
391 
392  void del (const std::string& file)
393  {
394  ftp_file_or_dir_action (file, "dele");
395  }
396 
397  void rmdir (const std::string& path)
398  {
399  ftp_file_or_dir_action (path, "rmd");
400  }
401 
402  void mkdir (const std::string& path)
403  {
404  ftp_file_or_dir_action (path, "mkd");
405  }
406 
407  void rename (const std::string& oldname, const std::string& newname)
408  {
409  struct curl_slist *slist = 0;
410 
411  unwind_protect frame;
412  frame.add_fcn (curl_slist_free_all, slist);
413 
414  std::string cmd = "rnfr " + oldname;
415  slist = curl_slist_append (slist, cmd.c_str ());
416  cmd = "rnto " + newname;
417  slist = curl_slist_append (slist, cmd.c_str ());
418  SETOPT (CURLOPT_POSTQUOTE, slist);
419 
420  perform ();
421  if (! good ())
422  return;
423 
424  SETOPT (CURLOPT_POSTQUOTE, 0);
425  }
426 
427  void put (const std::string& file, std::istream& is)
428  {
429  url = "ftp://" + host_or_url + "/" + file;
430  SETOPT (CURLOPT_URL, url.c_str ());
431  SETOPT (CURLOPT_UPLOAD, 1);
432  SETOPT (CURLOPT_NOBODY, 0);
433  std::istream& old_is = set_istream (is);
434 
435  perform ();
436  if (! good ())
437  return;
438 
439  set_istream (old_is);
440  SETOPT (CURLOPT_NOBODY, 1);
441  SETOPT (CURLOPT_UPLOAD, 0);
442  url = "ftp://" + host_or_url;
443  SETOPT (CURLOPT_URL, url.c_str ());
444  }
445 
446  void get (const std::string& file, std::ostream& os)
447  {
448  url = "ftp://" + host_or_url + "/" + file;
449  SETOPT (CURLOPT_URL, url.c_str ());
450  SETOPT (CURLOPT_NOBODY, 0);
451  std::ostream& old_os = set_ostream (os);
452 
453  perform ();
454  if (! good ())
455  return;
456 
457  set_ostream (old_os);
458  SETOPT (CURLOPT_NOBODY, 1);
459  url = "ftp://" + host_or_url;
460  SETOPT (CURLOPT_URL, url.c_str ());
461  }
462 
463  void dir (void)
464  {
465  url = "ftp://" + host_or_url + "/";
466  SETOPT (CURLOPT_URL, url.c_str ());
467  SETOPT (CURLOPT_NOBODY, 0);
468 
469  perform ();
470  if (! good ())
471  return;
472 
473  SETOPT (CURLOPT_NOBODY, 1);
474  url = "ftp://" + host_or_url;
475  SETOPT (CURLOPT_URL, url.c_str ());
476  }
477 
479  {
480  string_vector retval;
481 
482  std::ostringstream buf;
483  url = "ftp://" + host_or_url + "/";
484  SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (&buf));
485  SETOPTR (CURLOPT_URL, url.c_str ());
486  SETOPTR (CURLOPT_DIRLISTONLY, 1);
487  SETOPTR (CURLOPT_NOBODY, 0);
488 
489  perform ();
490  if (! good ())
491  return retval;
492 
493  SETOPTR (CURLOPT_NOBODY, 1);
494  url = "ftp://" + host_or_url;
495  SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (curr_ostream));
496  SETOPTR (CURLOPT_DIRLISTONLY, 0);
497  SETOPTR (CURLOPT_URL, url.c_str ());
498 
499  // Count number of directory entries
500  std::string str = buf.str ();
501  octave_idx_type n = 0;
502  size_t pos = 0;
503  while (true)
504  {
505  pos = str.find_first_of ('\n', pos);
506  if (pos == std::string::npos)
507  break;
508  pos++;
509  n++;
510  }
511  retval.resize (n);
512  pos = 0;
513  for (octave_idx_type i = 0; i < n; i++)
514  {
515  size_t newpos = str.find_first_of ('\n', pos);
516  if (newpos == std::string::npos)
517  break;
518 
519  retval(i) = str.substr(pos, newpos - pos);
520  pos = newpos + 1;
521  }
522 
523  return retval;
524  }
525 
526  void get_fileinfo (const std::string& filename, double& filesize,
527  time_t& filetime, bool& fileisdir)
528  {
529  std::string path = pwd ();
530 
531  url = "ftp://" + host_or_url + "/" + path + "/" + filename;
532  SETOPT (CURLOPT_URL, url.c_str ());
533  SETOPT (CURLOPT_FILETIME, 1);
534  SETOPT (CURLOPT_HEADERFUNCTION, throw_away);
535  SETOPT (CURLOPT_WRITEFUNCTION, throw_away);
536 
537  // FIXME
538  // The MDTM command fails for a directory on the servers I tested
539  // so this is a means of testing for directories. It also means
540  // I can't get the date of directories!
541 
542  perform ();
543  if (! good ())
544  {
545  fileisdir = true;
546  filetime = -1;
547  filesize = 0;
548 
549  return;
550  }
551 
552  fileisdir = false;
553  time_t ft;
554  curl_easy_getinfo (curl, CURLINFO_FILETIME, &ft);
555  filetime = ft;
556  double fs;
557  curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fs);
558  filesize = fs;
559 
560  SETOPT (CURLOPT_WRITEFUNCTION, write_data);
561  SETOPT (CURLOPT_HEADERFUNCTION, 0);
562  SETOPT (CURLOPT_FILETIME, 0);
563  url = "ftp://" + host_or_url;
564  SETOPT (CURLOPT_URL, url.c_str ());
565 
566  // The MDTM command seems to reset the path to the root with the
567  // servers I tested with, so cd again into the correct path. Make
568  // the path absolute so that this will work even with servers that
569  // don't end up in the root after an MDTM command.
570  cwd ("/" + path);
571  }
572 
573  std::string pwd (void)
574  {
575  std::string retval;
576 
577  struct curl_slist *slist = 0;
578 
579  unwind_protect frame;
580  frame.add_fcn (curl_slist_free_all, slist);
581 
582  slist = curl_slist_append (slist, "pwd");
583  SETOPTR (CURLOPT_POSTQUOTE, slist);
584  SETOPTR (CURLOPT_HEADERFUNCTION, write_data);
585 
586  std::ostringstream buf;
587  SETOPTR (CURLOPT_WRITEHEADER, static_cast<void *>(&buf));
588 
589  perform ();
590  if (! good ())
591  return retval;
592 
593  retval = buf.str ();
594 
595  // Can I assume that the path is alway in "" on the last line
596  size_t pos2 = retval.rfind ('"');
597  size_t pos1 = retval.rfind ('"', pos2 - 1);
598  retval = retval.substr (pos1 + 1, pos2 - pos1 - 1);
599 
600  SETOPTR (CURLOPT_HEADERFUNCTION, 0);
601  SETOPTR (CURLOPT_WRITEHEADER, 0);
602  SETOPTR (CURLOPT_POSTQUOTE, 0);
603 
604  return retval;
605  }
606 
607  void http_get (const Array<std::string>& param)
608  {
609  url = host_or_url;
610 
611  std::string query_string = form_query_string (param);
612 
613  if (! query_string.empty ())
614  url += "?" + query_string;
615 
616  SETOPT (CURLOPT_URL, url.c_str ());
617 
618  perform ();
619  }
620 
621  void http_post (const Array<std::string>& param)
622  {
623  SETOPT (CURLOPT_URL, host_or_url.c_str ());
624 
625  std::string query_string = form_query_string (param);
626 
627  SETOPT (CURLOPT_POSTFIELDS, query_string.c_str ());
628 
629  perform ();
630  }
631 
632  void http_action (const Array<std::string>& param, const std::string& action)
633  {
634  if (action.empty () || action == "get")
635  http_get (param);
636  else if (action == "post")
637  http_post (param);
638  else
639  {
640  ok = false;
641  errmsg = "curl_transfer: unknown http action";
642  }
643  }
644 
645 private:
646 
647  // Pointer to cURL object.
648  CURL *curl;
649 
650  // cURL error code.
651  CURLcode errnum;
652 
653  // The cURL library changed the curl_easy_setopt call to make an
654  // internal copy of string parameters in version 7.17.0. Prior
655  // versions only held a pointer to a string provided by the caller
656  // that must persist for the lifetime of the CURL handle.
657  //
658  // The associated API did not change, only the behavior of the library
659  // implementing the function call.
660  //
661  // To be compatible with any version of cURL, the caller must keep a
662  // copy of all string parameters associated with a CURL handle until
663  // the handle is released. The curl_handle::curl_handle_rep class
664  // contains the pointer to the CURL handle and so is the best
665  // candidate for storing the strings as well. (bug #36717)
666  std::string url;
667  std::string userpwd;
668 
669  // No copying!
670 
671  curl_transfer (const curl_transfer&);
672 
674 
675  void init (const std::string& user, const std::string& passwd,
676  std::istream& is, std::ostream& os)
677  {
678  // No data transfer by default
679  SETOPT (CURLOPT_NOBODY, 1);
680 
681  // Set the username and password
682  userpwd = user;
683  if (! passwd.empty ())
684  userpwd += ":" + passwd;
685  if (! userpwd.empty ())
686  SETOPT (CURLOPT_USERPWD, userpwd.c_str ());
687 
688  // Define our callback to get called when there's data to be written.
689  SETOPT (CURLOPT_WRITEFUNCTION, write_data);
690 
691  // Set a pointer to our struct to pass to the callback.
692  SETOPT (CURLOPT_WRITEDATA, static_cast<void*> (&os));
693 
694  // Define our callback to get called when there's data to be read
695  SETOPT (CURLOPT_READFUNCTION, read_data);
696 
697  // Set a pointer to our struct to pass to the callback.
698  SETOPT (CURLOPT_READDATA, static_cast<void*> (&is));
699 
700  // Follow redirects.
701  SETOPT (CURLOPT_FOLLOWLOCATION, true);
702 
703  // Don't use EPSV since connecting to sites that don't support it
704  // will hang for some time (3 minutes?) before moving on to try PASV
705  // instead.
706  SETOPT (CURLOPT_FTP_USE_EPSV, false);
707 
708  SETOPT (CURLOPT_NOPROGRESS, true);
709  SETOPT (CURLOPT_FAILONERROR, true);
710 
711  SETOPT (CURLOPT_POSTQUOTE, 0);
712  SETOPT (CURLOPT_QUOTE, 0);
713  }
714 
715  std::string form_query_string (const Array<std::string>& param)
716  {
717  std::ostringstream query;
718 
719  for (int i = 0; i < param.numel (); i += 2)
720  {
721  std::string name = param(i);
722  std::string text = param(i+1);
723 
724  // Encode strings.
725  char *enc_name = curl_easy_escape (curl, name.c_str (),
726  name.length ());
727  char *enc_text = curl_easy_escape (curl, text.c_str (),
728  text.length ());
729 
730  query << enc_name << "=" << enc_text;
731 
732  curl_free (enc_name);
733  curl_free (enc_text);
734 
735  if (i < param.numel ()-1)
736  query << "&";
737  }
738 
739  query.flush ();
740 
741  return query.str ();
742  }
743 
744  void ftp_file_or_dir_action (const std::string& file_or_dir,
745  const std::string& action)
746  {
747  struct curl_slist *slist = 0;
748 
749  unwind_protect frame;
750 
751  frame.add_fcn (curl_slist_free_all, slist);
752 
753  std::string cmd = action + " " + file_or_dir;
754 
755  slist = curl_slist_append (slist, cmd.c_str ());
756 
757  SETOPT (CURLOPT_POSTQUOTE, slist);
758 
759  perform ();
760 
761  if (! good ())
762  return;
763 
764  SETOPT (CURLOPT_POSTQUOTE, 0);
765  }
766 };
767 
768 #undef SETOPT
769 
770 #else
771 
772 static void
773 disabled_error (void)
774 {
775  (*current_liboctave_error_handler)
776  ("support for url transfers was disabled when Octave was built");
777 }
778 
779 #endif
780 
781 #if defined (HAVE_CURL)
782 # define REP_CLASS curl_transfer
783 #else
784 # define REP_CLASS base_url_transfer
785 #endif
786 
788 {
789 #if !defined (HAVE_CURL)
790  disabled_error ();
791 #endif
792 }
793 
794 url_transfer::url_transfer (const std::string& host, const std::string& user,
795  const std::string& passwd, std::ostream& os)
796  : rep (new REP_CLASS (host, user, passwd, os))
797 {
798 #if !defined (HAVE_CURL)
799  disabled_error ();
800 #endif
801 }
802 
803 url_transfer::url_transfer (const std::string& url, std::ostream& os)
804  : rep (new REP_CLASS (url, os))
805 {
806 #if !defined (HAVE_CURL)
807  disabled_error ();
808 #endif
809 }
810 
811 #undef REP_CLASS