tests/harness/backendmanager_remotetcp.cc

Go to the documentation of this file.
00001 
00004 /* Copyright (C) 2006,2007,2008 Olly Betts
00005  *
00006  * This program is free software; you can redistribute it and/or
00007  * modify it under the terms of the GNU General Public License as
00008  * published by the Free Software Foundation; either version 2 of the
00009  * License, or (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
00019  */
00020 
00021 #include <config.h>
00022 
00023 #include "backendmanager_remotetcp.h"
00024 
00025 #include <xapian.h>
00026 
00027 #include "safeerrno.h"
00028 #include <stdio.h>
00029 #include <cstring>
00030 
00031 #ifdef HAVE_FORK
00032 # include <signal.h>
00033 # include <sys/types.h>
00034 # include <sys/socket.h>
00035 # include <sys/wait.h>
00036 # include <unistd.h>
00037 // Some older systems had SIGCLD rather than SIGCHLD.
00038 # if !defined SIGCHLD && defined SIGCLD
00039 #  define SIGCHLD SIGCLD
00040 # endif
00041 #endif
00042 
00043 #ifdef __WIN32__
00044 # include "safefcntl.h"
00045 # include "safewindows.h"
00046 #endif
00047 
00048 #include "noreturn.h"
00049 #include "utils.h"
00050 
00051 #include <string>
00052 #include <vector>
00053 
00054 #ifdef HAVE_VALGRIND
00055 # include <valgrind/memcheck.h>
00056 #endif
00057 
00058 using namespace std;
00059 
00060 // We've had problems on some hosts which run tinderbox tests with "localhost"
00061 // not being set in /etc/hosts - using the IP address equivalent seems more
00062 // reliable.
00063 #define LOCALHOST "127.0.0.1"
00064 
00065 // Start at DEFAULT port and try higher ports until one isn't already in use.
00066 #define DEFAULT_PORT 1239
00067 
00068 #ifdef HAVE_FORK
00069 
00070 // We can't dynamically allocate memory for this because it confuses the leak
00071 // detector.  We only have 1-3 child fds open at once anyway, so a fixed size
00072 // array isn't a problem, and linear scanning isn't a problem.
00073 struct pid_fd {
00074     pid_t pid;
00075     int fd;
00076 };
00077 
00078 static pid_fd pid_to_fd[16];
00079 
00080 extern "C" void
00081 on_SIGCHLD(int /*sig*/)
00082 {
00083     int status;
00084     pid_t child;
00085     while ((child = waitpid(-1, &status, WNOHANG)) > 0) {
00086         for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
00087             if (pid_to_fd[i].pid == child) {
00088                 int fd = pid_to_fd[i].fd;
00089                 pid_to_fd[i].fd = -1;
00090                 pid_to_fd[i].pid = -1;
00091                 // NB close() *is* safe to use in a signal handler.
00092                 close(fd);
00093                 break;
00094             }
00095         }
00096     }
00097 }
00098 
00099 static int
00100 launch_xapian_tcpsrv(const string & args)
00101 {
00102     int port = DEFAULT_PORT;
00103 
00104     // We want to be able to get the exit status of the child process we fork
00105     // in xapian-tcpsrv doesn't start listening successfully.
00106     signal(SIGCHLD, SIG_DFL);
00107 try_next_port:
00108     string cmd = XAPIAN_TCPSRV" --one-shot --interface "LOCALHOST" --port " + om_tostring(port) + " " + args;
00109 #ifdef HAVE_VALGRIND
00110     if (RUNNING_ON_VALGRIND) cmd = "./runsrv " + cmd;
00111 #endif
00112     int fds[2];
00113     if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, fds) < 0) {
00114         string msg("Couldn't create socketpair: ");
00115         msg += strerror(errno);
00116         throw msg;
00117     }
00118 
00119     pid_t child = fork();
00120     if (child == 0) {
00121         // Child process.
00122         close(fds[0]);
00123         // Connect stdout and stderr to the socket.
00124         dup2(fds[1], 1);
00125         dup2(fds[1], 2);
00126         execl("/bin/sh", "/bin/sh", "-c", cmd.c_str(), (void*)NULL);
00127         _exit(-1);
00128     }
00129 
00130     close(fds[1]);
00131     if (child == -1) {
00132         // Couldn't fork.
00133         int fork_errno = errno;
00134         close(fds[0]);
00135         string msg("Couldn't fork: ");
00136         msg += strerror(fork_errno);
00137         throw msg;
00138     }
00139 
00140     // Parent process.
00141 
00142     // Wrap the file descriptor in a FILE * so we can read lines using fgets().
00143     FILE * fh = fdopen(fds[0], "r");
00144     if (fh == NULL) {
00145         string msg("Failed to run command '");
00146         msg += cmd;
00147         msg += "': ";
00148         msg += strerror(errno);
00149         throw msg;
00150     }
00151     string output;
00152     while (true) {
00153         char buf[256];
00154         if (fgets(buf, sizeof(buf), fh) == NULL) {
00155             fclose(fh);
00156             int status;
00157             if (waitpid(child, &status, 0) == -1) {
00158                 string msg("waitpid failed: ");
00159                 msg += strerror(errno);
00160                 throw msg;
00161             }
00162             if (++port < 65536 && status != 0) {
00163                 if (WIFEXITED(status) && WEXITSTATUS(status) == 69) {
00164                     // 69 is EX_UNAVAILABLE which xapian-tcpsrv exits
00165                     // with if (and only if) the port specified was
00166                     // in use.
00167                     goto try_next_port;
00168                 }
00169             }
00170             string msg("Failed to get 'Listening...' from command '");
00171             msg += cmd;
00172             msg += "' (output: ";
00173             msg += output;
00174             msg += ")";
00175             throw msg;
00176         }
00177         if (strcmp(buf, "Listening...\n") == 0) break;
00178         output += buf;
00179     }
00180 
00181     // dup() the fd we wrapped with fdopen() so we can keep it open so the
00182     // xapian-tcpsrv keeps running.
00183     int tracked_fd = dup(fds[0]);
00184 
00185     // We must fclose() the FILE* to avoid valgrind detecting memory leaks from
00186     // its buffers.
00187     fclose(fh);
00188 
00189     // Find a slot to track the pid->fd mapping in.  If we can't find a slot
00190     // it just means we'll leak the fd, so don't worry about that too much.
00191     for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
00192         if (pid_to_fd[i].pid == -1) {
00193             pid_to_fd[i].fd = tracked_fd;
00194             pid_to_fd[i].pid = child;
00195             break;
00196         }
00197     }
00198 
00199     // Set a signal handler to clean up the xapian-tcpsrv child process when it
00200     // finally exits.
00201     signal(SIGCHLD, on_SIGCHLD);
00202 
00203     return port;
00204 }
00205 
00206 #elif defined __WIN32__
00207 
00208 XAPIAN_NORETURN(static void win32_throw_error_string(const char * str));
00209 static void win32_throw_error_string(const char * str)
00210 {
00211     string msg(str);
00212     char * error = 0;
00213     DWORD len;
00214     len = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
00215                         0, GetLastError(), 0, (CHAR*)&error, 0, 0);
00216     if (error) {
00217         // Remove any trailing \r\n from output of FormatMessage.
00218         if (len >= 2 && error[len - 2] == '\r' && error[len - 1] == '\n')
00219             len -= 2;
00220         if (len) {
00221             msg += ": ";
00222             msg.append(error, len);
00223         }
00224         LocalFree(error);
00225     }
00226     throw msg;
00227 }
00228 
00229 // This implementation uses the WIN32 API to start xapian-tcpsrv as a child
00230 // process and read its output using a pipe.
00231 static int
00232 launch_xapian_tcpsrv(const string & args)
00233 {
00234     int port = DEFAULT_PORT;
00235 
00236 try_next_port:
00237     string cmd = XAPIAN_TCPSRV" --one-shot --interface "LOCALHOST" --port " + om_tostring(port) + " " + args;
00238 
00239     // Create a pipe so we can read stdout/stderr from the child process.
00240     HANDLE hRead, hWrite;
00241     if (!CreatePipe(&hRead, &hWrite, 0, 0))
00242         win32_throw_error_string("Couldn't create pipe");
00243 
00244     // Set the write handle to be inherited by the child process.
00245     SetHandleInformation(hWrite, HANDLE_FLAG_INHERIT, 1);
00246 
00247     // Create the child process.
00248     PROCESS_INFORMATION procinfo;
00249     memset(&procinfo, 0, sizeof(PROCESS_INFORMATION));
00250 
00251     STARTUPINFO startupinfo;
00252     memset(&startupinfo, 0, sizeof(STARTUPINFO));
00253     startupinfo.cb = sizeof(STARTUPINFO);
00254     startupinfo.hStdError = hWrite;
00255     startupinfo.hStdOutput = hWrite;
00256     startupinfo.hStdInput = INVALID_HANDLE_VALUE;
00257     startupinfo.dwFlags |= STARTF_USESTDHANDLES;
00258 
00259     // For some reason Windows wants a modifiable copy!
00260     BOOL ok;
00261     char * cmdline = strdup(cmd.c_str());
00262     ok = CreateProcess(0, cmdline, 0, 0, TRUE, 0, 0, 0, &startupinfo, &procinfo);
00263     free(cmdline);
00264     if (!ok)
00265         win32_throw_error_string("Couldn't create child process");
00266 
00267     CloseHandle(hWrite);
00268     CloseHandle(procinfo.hThread);
00269 
00270     string output;
00271     FILE *fh = fdopen(_open_osfhandle((intptr_t)hRead, O_RDONLY), "r");
00272     while (true) {
00273         char buf[256];
00274         if (fgets(buf, sizeof(buf), fh) == NULL) {
00275             fclose(fh);
00276             DWORD rc;
00277             // This doesn't seem to be necessary on the machine I tested on,
00278             // but I guess it could be on a slow machine...
00279             while (GetExitCodeProcess(procinfo.hProcess, &rc) && rc == STILL_ACTIVE) {
00280                 Sleep(100);
00281             }
00282             CloseHandle(procinfo.hProcess);
00283             if (++port < 65536 && rc == 69) {
00284                 // 69 is EX_UNAVAILABLE which xapian-tcpsrv exits
00285                 // with if (and only if) the port specified was
00286                 // in use.
00287                 goto try_next_port;
00288             }
00289             string msg("Failed to get 'Listening...' from command '");
00290             msg += cmd;
00291             msg += "' (output: ";
00292             msg += output;
00293             msg += ")";
00294             throw msg;
00295         }
00296         if (strcmp(buf, "Listening...\r\n") == 0) break;
00297         output += buf;
00298     }
00299     fclose(fh);
00300 
00301     return port;
00302 }
00303 
00304 #else
00305 # error Neither HAVE_FORK nor __WIN32__ is defined
00306 #endif
00307 
00308 BackendManagerRemoteTcp::~BackendManagerRemoteTcp() { }
00309 
00310 const char *
00311 BackendManagerRemoteTcp::get_dbtype() const
00312 {
00313     return "remotetcp";
00314 }
00315 
00316 Xapian::Database
00317 BackendManagerRemoteTcp::get_database(const vector<string> & files)
00318 {
00319     // Default to a long (5 minute) timeout so that tests won't fail just
00320     // because the host is slow or busy.
00321     return BackendManagerRemoteTcp::get_remote_database(files, 300000);
00322 }
00323 
00324 Xapian::Database
00325 BackendManagerRemoteTcp::get_database(const string & file)
00326 {
00327     return BackendManagerRemoteTcp::get_database(vector<string>(1, file));
00328 }
00329 
00330 Xapian::WritableDatabase
00331 BackendManagerRemoteTcp::get_writable_database(const string & name,
00332                                                const string & file)
00333 {
00334     last_wdb_name = name;
00335 
00336     // Default to a long (5 minute) timeout so that tests won't fail just
00337     // because the host is slow or busy.
00338     string args = "-t300000 --writable ";
00339 
00340 #ifdef XAPIAN_HAS_FLINT_BACKEND
00341     (void)getwritedb_flint(name, vector<string>(1, file));
00342     args += ".flint/";
00343 #else
00344     (void)getwritedb_quartz(vector<string>(1, file));
00345     args += ".quartz/";
00346 #endif
00347     args += name;
00348 
00349     int port = launch_xapian_tcpsrv(args);
00350     return Xapian::Remote::open_writable(LOCALHOST, port);
00351 }
00352 
00353 Xapian::Database
00354 BackendManagerRemoteTcp::get_remote_database(const vector<string> & files,
00355                                              unsigned int timeout)
00356 {
00357     string args = "-t";
00358     args += om_tostring(timeout);
00359     args += ' ';
00360 #ifdef XAPIAN_HAS_FLINT_BACKEND
00361     args += createdb_flint(files);
00362 #else
00363     args += createdb_quartz(files);
00364 #endif
00365 
00366     int port = launch_xapian_tcpsrv(args);
00367     return Xapian::Remote::open(LOCALHOST, port);
00368 }
00369 
00370 Xapian::Database
00371 BackendManagerRemoteTcp::get_writable_database_as_database()
00372 {
00373     string args = "-t300000 ";
00374 #ifdef XAPIAN_HAS_FLINT_BACKEND
00375     args += ".flint/";
00376 #else
00377     args += ".quartz/";
00378 #endif
00379     args += last_wdb_name;
00380 
00381     int port = launch_xapian_tcpsrv(args);
00382     return Xapian::Remote::open(LOCALHOST, port);
00383 }
00384 
00385 Xapian::WritableDatabase
00386 BackendManagerRemoteTcp::get_writable_database_again()
00387 {
00388     string args = "-t300000 --writable ";
00389 #ifdef XAPIAN_HAS_FLINT_BACKEND
00390     args += ".flint/";
00391 #else
00392     args += ".quartz/";
00393 #endif
00394     args += last_wdb_name;
00395 
00396     int port = launch_xapian_tcpsrv(args);
00397     return Xapian::Remote::open_writable(LOCALHOST, port);
00398 }

Documentation for Xapian (version 1.0.10).
Generated on 24 Dec 2008 by Doxygen 1.5.2.