MemcachedClient.php

Go to the documentation of this file.

<?php
// {{{ requirements
// }}}

// {{{ class MWMemcached
class MWMemcached {
    // {{{ properties
    // {{{ public

    // {{{ constants
    // {{{ flags

    const SERIALIZED = 1;

    const COMPRESSED = 2;

    // }}}

    const COMPRESSION_SAVINGS = 0.20;

    // }}}

    var $stats;

    // }}}
    // {{{ private

    var $_cache_sock;

    var $_debug;

    var $_host_dead;

    var $_have_zlib;

    var $_compress_enable;

    var $_compress_threshold;

    var $_persistent;

    var $_single_sock;

    var $_servers;

    var $_buckets;

    var $_bucketcount;

    var $_active;

    var $_timeout_seconds;

    var $_timeout_microseconds;

    var $_connect_timeout;

    var $_connect_attempts;

    // }}}
    // }}}
    // {{{ methods
    // {{{ public functions
    // {{{ memcached()

    public function __construct( $args ) {
        $this->set_servers( isset( $args['servers'] ) ? $args['servers'] : array() );
        $this->_debug = isset( $args['debug'] ) ? $args['debug'] : false;
        $this->stats = array();
        $this->_compress_threshold = isset( $args['compress_threshold'] ) ? $args['compress_threshold'] : 0;
        $this->_persistent = isset( $args['persistent'] ) ? $args['persistent'] : false;
        $this->_compress_enable = true;
        $this->_have_zlib = function_exists( 'gzcompress' );

        $this->_cache_sock = array();
        $this->_host_dead = array();

        $this->_timeout_seconds = 0;
        $this->_timeout_microseconds = isset( $args['timeout'] ) ? $args['timeout'] : 500000;

        $this->_connect_timeout = isset( $args['connect_timeout'] ) ? $args['connect_timeout'] : 0.1;
        $this->_connect_attempts = 2;
    }

    // }}}
    // {{{ add()

    public function add( $key, $val, $exp = 0 ) {
        return $this->_set( 'add', $key, $val, $exp );
    }

    // }}}
    // {{{ decr()

    public function decr( $key, $amt = 1 ) {
        return $this->_incrdecr( 'decr', $key, $amt );
    }

    // }}}
    // {{{ delete()

    public function delete( $key, $time = 0 ) {
        if ( !$this->_active ) {
            return false;
        }

        $sock = $this->get_sock( $key );
        if ( !is_resource( $sock ) ) {
            return false;
        }

        $key = is_array( $key ) ? $key[1] : $key;

        if ( isset( $this->stats['delete'] ) ) {
            $this->stats['delete']++;
        } else {
            $this->stats['delete'] = 1;
        }
        $cmd = "delete $key $time\r\n";
        if ( !$this->_fwrite( $sock, $cmd ) ) {
            return false;
        }
        $res = $this->_fgets( $sock );

        if ( $this->_debug ) {
            $this->_debugprint( sprintf( "MemCache: delete %s (%s)\n", $key, $res ) );
        }

        if ( $res == "DELETED" || $res == "NOT_FOUND" ) {
            return true;
        }

        return false;
    }

    public function lock( $key, $timeout = 0 ) {
        /* stub */
        return true;
    }

    public function unlock( $key ) {
        /* stub */
        return true;
    }

    // }}}
    // {{{ disconnect_all()

    public function disconnect_all() {
        foreach ( $this->_cache_sock as $sock ) {
            fclose( $sock );
        }

        $this->_cache_sock = array();
    }

    // }}}
    // {{{ enable_compress()

    public function enable_compress( $enable ) {
        $this->_compress_enable = $enable;
    }

    // }}}
    // {{{ forget_dead_hosts()

    public function forget_dead_hosts() {
        $this->_host_dead = array();
    }

    // }}}
    // {{{ get()

    public function get( $key, &$casToken = null ) {
        wfProfileIn( __METHOD__ );

        if ( $this->_debug ) {
            $this->_debugprint( "get($key)\n" );
        }

        if ( !$this->_active ) {
            wfProfileOut( __METHOD__ );
            return false;
        }

        $sock = $this->get_sock( $key );

        if ( !is_resource( $sock ) ) {
            wfProfileOut( __METHOD__ );
            return false;
        }

        $key = is_array( $key ) ? $key[1] : $key;
        if ( isset( $this->stats['get'] ) ) {
            $this->stats['get']++;
        } else {
            $this->stats['get'] = 1;
        }

        $cmd = "gets $key\r\n";
        if ( !$this->_fwrite( $sock, $cmd ) ) {
            wfProfileOut( __METHOD__ );
            return false;
        }

        $val = array();
        $this->_load_items( $sock, $val, $casToken );

        if ( $this->_debug ) {
            foreach ( $val as $k => $v ) {
                $this->_debugprint( sprintf( "MemCache: sock %s got %s\n", serialize( $sock ), $k ) );
            }
        }

        $value = false;
        if ( isset( $val[$key] ) ) {
            $value = $val[$key];
        }
        wfProfileOut( __METHOD__ );
        return $value;
    }

    // }}}
    // {{{ get_multi()

    public function get_multi( $keys ) {
        if ( !$this->_active ) {
            return false;
        }

        if ( isset( $this->stats['get_multi'] ) ) {
            $this->stats['get_multi']++;
        } else {
            $this->stats['get_multi'] = 1;
        }
        $sock_keys = array();
        $socks = array();
        foreach ( $keys as $key ) {
            $sock = $this->get_sock( $key );
            if ( !is_resource( $sock ) ) {
                continue;
            }
            $key = is_array( $key ) ? $key[1] : $key;
            if ( !isset( $sock_keys[$sock] ) ) {
                $sock_keys[intval( $sock )] = array();
                $socks[] = $sock;
            }
            $sock_keys[intval( $sock )][] = $key;
        }

        $gather = array();
        // Send out the requests
        foreach ( $socks as $sock ) {
            $cmd = 'gets';
            foreach ( $sock_keys[intval( $sock )] as $key ) {
                $cmd .= ' ' . $key;
            }
            $cmd .= "\r\n";

            if ( $this->_fwrite( $sock, $cmd ) ) {
                $gather[] = $sock;
            }
        }

        // Parse responses
        $val = array();
        foreach ( $gather as $sock ) {
            $this->_load_items( $sock, $val, $casToken );
        }

        if ( $this->_debug ) {
            foreach ( $val as $k => $v ) {
                $this->_debugprint( sprintf( "MemCache: got %s\n", $k ) );
            }
        }

        return $val;
    }

    // }}}
    // {{{ incr()

    public function incr( $key, $amt = 1 ) {
        return $this->_incrdecr( 'incr', $key, $amt );
    }

    // }}}
    // {{{ replace()

    public function replace( $key, $value, $exp = 0 ) {
        return $this->_set( 'replace', $key, $value, $exp );
    }

    // }}}
    // {{{ run_command()

    public function run_command( $sock, $cmd ) {
        if ( !is_resource( $sock ) ) {
            return array();
        }

        if ( !$this->_fwrite( $sock, $cmd ) ) {
            return array();
        }

        $ret = array();
        while ( true ) {
            $res = $this->_fgets( $sock );
            $ret[] = $res;
            if ( preg_match( '/^END/', $res ) ) {
                break;
            }
            if ( strlen( $res ) == 0 ) {
                break;
            }
        }
        return $ret;
    }

    // }}}
    // {{{ set()

    public function set( $key, $value, $exp = 0 ) {
        return $this->_set( 'set', $key, $value, $exp );
    }

    // }}}
    // {{{ cas()

    public function cas( $casToken, $key, $value, $exp = 0 ) {
        return $this->_set( 'cas', $key, $value, $exp, $casToken );
    }

    // }}}
    // {{{ set_compress_threshold()

    public function set_compress_threshold( $thresh ) {
        $this->_compress_threshold = $thresh;
    }

    // }}}
    // {{{ set_debug()

    public function set_debug( $dbg ) {
        $this->_debug = $dbg;
    }

    // }}}
    // {{{ set_servers()

    public function set_servers( $list ) {
        $this->_servers = $list;
        $this->_active = count( $list );
        $this->_buckets = null;
        $this->_bucketcount = 0;

        $this->_single_sock = null;
        if ( $this->_active == 1 ) {
            $this->_single_sock = $this->_servers[0];
        }
    }

    public function set_timeout( $seconds, $microseconds ) {
        $this->_timeout_seconds = $seconds;
        $this->_timeout_microseconds = $microseconds;
    }

    // }}}
    // }}}
    // {{{ private methods
    // {{{ _close_sock()

    function _close_sock( $sock ) {
        $host = array_search( $sock, $this->_cache_sock );
        fclose( $this->_cache_sock[$host] );
        unset( $this->_cache_sock[$host] );
    }

    // }}}
    // {{{ _connect_sock()

    function _connect_sock( &$sock, $host ) {
        list( $ip, $port ) = explode( ':', $host );
        $sock = false;
        $timeout = $this->_connect_timeout;
        $errno = $errstr = null;
        for ( $i = 0; !$sock && $i < $this->_connect_attempts; $i++ ) {
            wfSuppressWarnings();
            if ( $this->_persistent == 1 ) {
                $sock = pfsockopen( $ip, $port, $errno, $errstr, $timeout );
            } else {
                $sock = fsockopen( $ip, $port, $errno, $errstr, $timeout );
            }
            wfRestoreWarnings();
        }
        if ( !$sock ) {
            $this->_error_log( "Error connecting to $host: $errstr\n" );
            $this->_dead_host( $host );
            return false;
        }

        // Initialise timeout
        stream_set_timeout( $sock, $this->_timeout_seconds, $this->_timeout_microseconds );

        // If the connection was persistent, flush the read buffer in case there
        // was a previous incomplete request on this connection
        if ( $this->_persistent ) {
            $this->_flush_read_buffer( $sock );
        }
        return true;
    }

    // }}}
    // {{{ _dead_sock()

    function _dead_sock( $sock ) {
        $host = array_search( $sock, $this->_cache_sock );
        $this->_dead_host( $host );
    }

    function _dead_host( $host ) {
        $parts = explode( ':', $host );
        $ip = $parts[0];
        $this->_host_dead[$ip] = time() + 30 + intval( rand( 0, 10 ) );
        $this->_host_dead[$host] = $this->_host_dead[$ip];
        unset( $this->_cache_sock[$host] );
    }

    // }}}
    // {{{ get_sock()

    function get_sock( $key ) {
        if ( !$this->_active ) {
            return false;
        }

        if ( $this->_single_sock !== null ) {
            return $this->sock_to_host( $this->_single_sock );
        }

        $hv = is_array( $key ) ? intval( $key[0] ) : $this->_hashfunc( $key );
        if ( $this->_buckets === null ) {
            $bu = array();
            foreach ( $this->_servers as $v ) {
                if ( is_array( $v ) ) {
                    for ( $i = 0; $i < $v[1]; $i++ ) {
                        $bu[] = $v[0];
                    }
                } else {
                    $bu[] = $v;
                }
            }
            $this->_buckets = $bu;
            $this->_bucketcount = count( $bu );
        }

        $realkey = is_array( $key ) ? $key[1] : $key;
        for ( $tries = 0; $tries < 20; $tries++ ) {
            $host = $this->_buckets[$hv % $this->_bucketcount];
            $sock = $this->sock_to_host( $host );
            if ( is_resource( $sock ) ) {
                return $sock;
            }
            $hv = $this->_hashfunc( $hv . $realkey );
        }

        return false;
    }

    // }}}
    // {{{ _hashfunc()

    function _hashfunc( $key ) {
        # Hash function must be in [0,0x7ffffff]
        # We take the first 31 bits of the MD5 hash, which unlike the hash
        # function used in a previous version of this client, works
        return hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff;
    }

    // }}}
    // {{{ _incrdecr()

    function _incrdecr( $cmd, $key, $amt = 1 ) {
        if ( !$this->_active ) {
            return null;
        }

        $sock = $this->get_sock( $key );
        if ( !is_resource( $sock ) ) {
            return null;
        }

        $key = is_array( $key ) ? $key[1] : $key;
        if ( isset( $this->stats[$cmd] ) ) {
            $this->stats[$cmd]++;
        } else {
            $this->stats[$cmd] = 1;
        }
        if ( !$this->_fwrite( $sock, "$cmd $key $amt\r\n" ) ) {
            return null;
        }

        $line = $this->_fgets( $sock );
        $match = array();
        if ( !preg_match( '/^(\d+)/', $line, $match ) ) {
            return null;
        }
        return $match[1];
    }

    // }}}
    // {{{ _load_items()

    function _load_items( $sock, &$ret, &$casToken = null ) {
        $results = array();

        while ( 1 ) {
            $decl = $this->_fgets( $sock );

            if ( $decl === false ) {
                /*
                 * If nothing can be read, something is wrong because we know exactly when
                 * to stop reading (right after "END") and we return right after that.
                 */
                return false;
            } elseif ( preg_match( '/^VALUE (\S+) (\d+) (\d+) (\d+)$/', $decl, $match ) ) {
                /*
                 * Read all data returned. This can be either one or multiple values.
                 * Save all that data (in an array) to be processed later: we'll first
                 * want to continue reading until "END" before doing anything else,
                 * to make sure that we don't leave our client in a state where it's
                 * output is not yet fully read.
                 */
                $results[] = array(
                    $match[1], // rkey
                    $match[2], // flags
                    $match[3], // len
                    $match[4], // casToken
                    $this->_fread( $sock, $match[3] + 2 ), // data
                );
            } elseif ( $decl == "END" ) {
                if ( count( $results ) == 0 ) {
                    return false;
                }

                foreach ( $results as $vars ) {
                    list( $rkey, $flags, $len, $casToken, $data ) = $vars;

                    if ( $data === false || substr( $data, -2 ) !== "\r\n" ) {
                        $this->_handle_error( $sock,
                            'line ending missing from data block from $1' );
                        return false;
                    }
                    $data = substr( $data, 0, -2 );
                    $ret[$rkey] = $data;

                    if ( $this->_have_zlib && $flags & self::COMPRESSED ) {
                        $ret[$rkey] = gzuncompress( $ret[$rkey] );
                    }

                    /*
                     * This unserialize is the exact reason that we only want to
                     * process data after having read until "END" (instead of doing
                     * this right away): "unserialize" can trigger outside code:
                     * in the event that $ret[$rkey] is a serialized object,
                     * unserializing it will trigger __wakeup() if present. If that
                     * function attempted to read from memcached (while we did not
                     * yet read "END"), these 2 calls would collide.
                     */
                    if ( $flags & self::SERIALIZED ) {
                        $ret[$rkey] = unserialize( $ret[$rkey] );
                    }
                }

                return true;
            } else {
                $this->_handle_error( $sock, 'Error parsing response from $1' );
                return false;
            }
        }
    }

    // }}}
    // {{{ _set()

    function _set( $cmd, $key, $val, $exp, $casToken = null ) {
        if ( !$this->_active ) {
            return false;
        }

        $sock = $this->get_sock( $key );
        if ( !is_resource( $sock ) ) {
            return false;
        }

        if ( isset( $this->stats[$cmd] ) ) {
            $this->stats[$cmd]++;
        } else {
            $this->stats[$cmd] = 1;
        }

        $flags = 0;

        if ( !is_scalar( $val ) ) {
            $val = serialize( $val );
            $flags |= self::SERIALIZED;
            if ( $this->_debug ) {
                $this->_debugprint( sprintf( "client: serializing data as it is not scalar\n" ) );
            }
        }

        $len = strlen( $val );

        if ( $this->_have_zlib && $this->_compress_enable &&
            $this->_compress_threshold && $len >= $this->_compress_threshold )
        {
            $c_val = gzcompress( $val, 9 );
            $c_len = strlen( $c_val );

            if ( $c_len < $len * ( 1 - self::COMPRESSION_SAVINGS ) ) {
                if ( $this->_debug ) {
                    $this->_debugprint( sprintf( "client: compressing data; was %d bytes is now %d bytes\n", $len, $c_len ) );
                }
                $val = $c_val;
                $len = $c_len;
                $flags |= self::COMPRESSED;
            }
        }

        $command = "$cmd $key $flags $exp $len";
        if ( $casToken ) {
            $command .= " $casToken";
        }

        if ( !$this->_fwrite( $sock, "$command\r\n$val\r\n" ) ) {
            return false;
        }

        $line = $this->_fgets( $sock );

        if ( $this->_debug ) {
            $this->_debugprint( sprintf( "%s %s (%s)\n", $cmd, $key, $line ) );
        }
        if ( $line == "STORED" ) {
            return true;
        }
        return false;
    }

    // }}}
    // {{{ sock_to_host()

    function sock_to_host( $host ) {
        if ( isset( $this->_cache_sock[$host] ) ) {
            return $this->_cache_sock[$host];
        }

        $sock = null;
        $now = time();
        list( $ip, /* $port */) = explode( ':', $host );
        if ( isset( $this->_host_dead[$host] ) && $this->_host_dead[$host] > $now ||
            isset( $this->_host_dead[$ip] ) && $this->_host_dead[$ip] > $now
        ) {
            return null;
        }

        if ( !$this->_connect_sock( $sock, $host ) ) {
            return null;
        }

        // Do not buffer writes
        stream_set_write_buffer( $sock, 0 );

        $this->_cache_sock[$host] = $sock;

        return $this->_cache_sock[$host];
    }

    function _debugprint( $text ) {
        wfDebugLog( 'memcached', $text );
    }

    function _error_log( $text ) {
        wfDebugLog( 'memcached-serious', "Memcached error: $text" );
    }

    function _fwrite( $sock, $buf ) {
        $bytesWritten = 0;
        $bufSize = strlen( $buf );
        while ( $bytesWritten < $bufSize ) {
            $result = fwrite( $sock, $buf );
            $data = stream_get_meta_data( $sock );
            if ( $data['timed_out'] ) {
                $this->_handle_error( $sock, 'timeout writing to $1' );
                return false;
            }
            // Contrary to the documentation, fwrite() returns zero on error in PHP 5.3.
            if ( $result === false || $result === 0 ) {
                $this->_handle_error( $sock, 'error writing to $1' );
                return false;
            }
            $bytesWritten += $result;
        }

        return true;
    }

    function _handle_error( $sock, $msg ) {
        $peer = stream_socket_get_name( $sock, true  );
        if ( strval( $peer ) === '' ) {
            $peer = array_search( $sock, $this->_cache_sock );
            if ( $peer === false ) {
                $peer = '[unknown host]';
            }
        }
        $msg = str_replace( '$1', $peer, $msg );
        $this->_error_log( "$msg\n" );
        $this->_dead_sock( $sock );
    }

    function _fread( $sock, $len ) {
        $buf = '';
        while ( $len > 0 ) {
            $result = fread( $sock, $len );
            $data = stream_get_meta_data( $sock );
            if ( $data['timed_out'] ) {
                $this->_handle_error( $sock, 'timeout reading from $1' );
                return false;
            }
            if ( $result === false ) {
                $this->_handle_error( $sock, 'error reading buffer from $1' );
                return false;
            }
            if ( $result === '' ) {
                // This will happen if the remote end of the socket is shut down
                $this->_handle_error( $sock, 'unexpected end of file reading from $1' );
                return false;
            }
            $len -= strlen( $result );
            $buf .= $result;
        }
        return $buf;
    }

    function _fgets( $sock ) {
        $result = fgets( $sock );
        // fgets() may return a partial line if there is a select timeout after
        // a successful recv(), so we have to check for a timeout even if we
        // got a string response.
        $data = stream_get_meta_data( $sock );
        if ( $data['timed_out'] ) {
            $this->_handle_error( $sock, 'timeout reading line from $1' );
            return false;
        }
        if ( $result === false ) {
            $this->_handle_error( $sock, 'error reading line from $1' );
            return false;
        }
        if ( substr( $result, -2 ) === "\r\n" ) {
            $result = substr( $result, 0, -2 );
        } elseif ( substr( $result, -1 ) === "\n" ) {
            $result = substr( $result, 0, -1 );
        } else {
            $this->_handle_error( $sock, 'line ending missing in response from $1' );
            return false;
        }
        return $result;
    }

    function _flush_read_buffer( $f ) {
        if ( !is_resource( $f ) ) {
            return;
        }
        $r = array( $f );
        $w = null;
        $e = null;
        $n = stream_select( $r, $w, $e, 0, 0 );
        while ( $n == 1 && !feof( $f ) ) {
            fread( $f, 1024 );
            $r = array( $f );
            $w = null;
            $e = null;
            $n = stream_select( $r, $w, $e, 0, 0 );
        }
    }

    // }}}
    // }}}
    // }}}
}

// }}}

class MemCachedClientforWiki extends MWMemcached {
}