MemcachedClient.php

Go to the documentation of this file.

<?php
// {{{ requirements
// }}}

// {{{ class MWMemcached
class MWMemcached {
        // {{{ properties
        // {{{ public

        // {{{ constants
        // {{{ flags

        const SERIALIZED = 1;

        const COMPRESSED = 2;

        // }}}

        const COMPRESSION_SAVINGS = 0.20;

        // }}}


        var $stats;

        // }}}
        // {{{ private

        var $_cache_sock;

        var $_debug;

        var $_host_dead;

        var $_have_zlib;

        var $_compress_enable;

        var $_compress_threshold;

        var $_persistent;

        var $_single_sock;

        var $_servers;

        var $_buckets;

        var $_bucketcount;

        var $_active;

        var $_timeout_seconds;

        var $_timeout_microseconds;

        var $_connect_timeout;

        var $_connect_attempts;

        // }}}
        // }}}
        // {{{ methods
        // {{{ public functions
        // {{{ memcached()

        public function __construct( $args ) {
                $this->set_servers( isset( $args['servers'] ) ? $args['servers'] : array() );
                $this->_debug = isset( $args['debug'] ) ? $args['debug'] : false;
                $this->stats = array();
                $this->_compress_threshold = isset( $args['compress_threshold'] ) ? $args['compress_threshold'] : 0;
                $this->_persistent = isset( $args['persistent'] ) ? $args['persistent'] : false;
                $this->_compress_enable = true;
                $this->_have_zlib = function_exists( 'gzcompress' );

                $this->_cache_sock = array();
                $this->_host_dead = array();

                $this->_timeout_seconds = 0;
                $this->_timeout_microseconds = isset( $args['timeout'] ) ? $args['timeout'] : 500000;

                $this->_connect_timeout = isset( $args['connect_timeout'] ) ? $args['connect_timeout'] : 0.1;
                $this->_connect_attempts = 2;
        }

        // }}}
        // {{{ add()

        public function add( $key, $val, $exp = 0 ) {
                return $this->_set( 'add', $key, $val, $exp );
        }

        // }}}
        // {{{ decr()

        public function decr( $key, $amt = 1 ) {
                return $this->_incrdecr( 'decr', $key, $amt );
        }

        // }}}
        // {{{ delete()

        public function delete( $key, $time = 0 ) {
                if ( !$this->_active ) {
                        return false;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return false;
                }

                $key = is_array( $key ) ? $key[1] : $key;

                if ( isset( $this->stats['delete'] ) ) {
                        $this->stats['delete']++;
                } else {
                        $this->stats['delete'] = 1;
                }
                $cmd = "delete $key $time\r\n";
                if( !$this->_fwrite( $sock, $cmd ) ) {
                        return false;
                }
                $res = $this->_fgets( $sock );

                if ( $this->_debug ) {
                        $this->_debugprint( sprintf( "MemCache: delete %s (%s)\n", $key, $res ) );
                }

                if ( $res == "DELETED" || $res == "NOT_FOUND" ) {
                        return true;
                }

                return false;
        }

        public function lock( $key, $timeout = 0 ) {
                /* stub */
                return true;
        }

        public function unlock( $key ) {
                /* stub */
                return true;
        }

        // }}}
        // {{{ disconnect_all()

        public function disconnect_all() {
                foreach ( $this->_cache_sock as $sock ) {
                        fclose( $sock );
                }

                $this->_cache_sock = array();
        }

        // }}}
        // {{{ enable_compress()

        public function enable_compress( $enable ) {
                $this->_compress_enable = $enable;
        }

        // }}}
        // {{{ forget_dead_hosts()

        public function forget_dead_hosts() {
                $this->_host_dead = array();
        }

        // }}}
        // {{{ get()

        public function get( $key ) {
                wfProfileIn( __METHOD__ );

                if ( $this->_debug ) {
                        $this->_debugprint( "get($key)\n" );
                }

                if ( !$this->_active ) {
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $sock = $this->get_sock( $key );

                if ( !is_resource( $sock ) ) {
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $key = is_array( $key ) ? $key[1] : $key;
                if ( isset( $this->stats['get'] ) ) {
                        $this->stats['get']++;
                } else {
                        $this->stats['get'] = 1;
                }

                $cmd = "get $key\r\n";
                if ( !$this->_fwrite( $sock, $cmd ) ) {
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $val = array();
                $this->_load_items( $sock, $val );

                if ( $this->_debug ) {
                        foreach ( $val as $k => $v ) {
                                $this->_debugprint( sprintf( "MemCache: sock %s got %s\n", serialize( $sock ), $k ) );
                        }
                }

                $value = false;
                if ( isset( $val[$key] ) ) {
                        $value = $val[$key];
                }
                wfProfileOut( __METHOD__ );
                return $value;
        }

        // }}}
        // {{{ get_multi()

        public function get_multi( $keys ) {
                if ( !$this->_active ) {
                        return false;
                }

                if ( isset( $this->stats['get_multi'] ) ) {
                        $this->stats['get_multi']++;
                } else {
                        $this->stats['get_multi'] = 1;
                }
                $sock_keys = array();
                $socks = array();
                foreach ( $keys as $key ) {
                        $sock = $this->get_sock( $key );
                        if ( !is_resource( $sock ) ) {
                                continue;
                        }
                        $key = is_array( $key ) ? $key[1] : $key;
                        if ( !isset( $sock_keys[$sock] ) ) {
                                $sock_keys[ intval( $sock ) ] = array();
                                $socks[] = $sock;
                        }
                        $sock_keys[ intval( $sock ) ][] = $key;
                }

                $gather = array();
                // Send out the requests
                foreach ( $socks as $sock ) {
                        $cmd = 'get';
                        foreach ( $sock_keys[ intval( $sock ) ] as $key ) {
                                $cmd .= ' ' . $key;
                        }
                        $cmd .= "\r\n";

                        if ( $this->_fwrite( $sock, $cmd ) ) {
                                $gather[] = $sock;
                        }
                }

                // Parse responses
                $val = array();
                foreach ( $gather as $sock ) {
                        $this->_load_items( $sock, $val );
                }

                if ( $this->_debug ) {
                        foreach ( $val as $k => $v ) {
                                $this->_debugprint( sprintf( "MemCache: got %s\n", $k ) );
                        }
                }

                return $val;
        }

        // }}}
        // {{{ incr()

        public function incr( $key, $amt = 1 ) {
                return $this->_incrdecr( 'incr', $key, $amt );
        }

        // }}}
        // {{{ replace()

        public function replace( $key, $value, $exp = 0 ) {
                return $this->_set( 'replace', $key, $value, $exp );
        }

        // }}}
        // {{{ run_command()

        public function run_command( $sock, $cmd ) {
                if ( !is_resource( $sock ) ) {
                        return array();
                }

                if ( !$this->_fwrite( $sock, $cmd ) ) {
                        return array();
                }

                $ret = array();
                while ( true ) {
                        $res = $this->_fgets( $sock );
                        $ret[] = $res;
                        if ( preg_match( '/^END/', $res ) ) {
                                break;
                        }
                        if ( strlen( $res ) == 0 ) {
                                break;
                        }
                }
                return $ret;
        }

        // }}}
        // {{{ set()

        public function set( $key, $value, $exp = 0 ) {
                return $this->_set( 'set', $key, $value, $exp );
        }

        // }}}
        // {{{ set_compress_threshold()

        public function set_compress_threshold( $thresh ) {
                $this->_compress_threshold = $thresh;
        }

        // }}}
        // {{{ set_debug()

        public function set_debug( $dbg ) {
                $this->_debug = $dbg;
        }

        // }}}
        // {{{ set_servers()

        public function set_servers( $list ) {
                $this->_servers = $list;
                $this->_active = count( $list );
                $this->_buckets = null;
                $this->_bucketcount = 0;

                $this->_single_sock = null;
                if ( $this->_active == 1 ) {
                        $this->_single_sock = $this->_servers[0];
                }
        }

        public function set_timeout( $seconds, $microseconds ) {
                $this->_timeout_seconds = $seconds;
                $this->_timeout_microseconds = $microseconds;
        }

        // }}}
        // }}}
        // {{{ private methods
        // {{{ _close_sock()

        function _close_sock( $sock ) {
                $host = array_search( $sock, $this->_cache_sock );
                fclose( $this->_cache_sock[$host] );
                unset( $this->_cache_sock[$host] );
        }

        // }}}
        // {{{ _connect_sock()

        function _connect_sock( &$sock, $host ) {
                list( $ip, $port ) = explode( ':', $host );
                $sock = false;
                $timeout = $this->_connect_timeout;
                $errno = $errstr = null;
                for( $i = 0; !$sock && $i < $this->_connect_attempts; $i++ ) {
                        wfSuppressWarnings();
                        if ( $this->_persistent == 1 ) {
                                $sock = pfsockopen( $ip, $port, $errno, $errstr, $timeout );
                        } else {
                                $sock = fsockopen( $ip, $port, $errno, $errstr, $timeout );
                        }
                        wfRestoreWarnings();
                }
                if ( !$sock ) {
                        $this->_error_log( "Error connecting to $host: $errstr\n" );
                        $this->_dead_host( $host );
                        return false;
                }

                // Initialise timeout
                stream_set_timeout( $sock, $this->_timeout_seconds, $this->_timeout_microseconds );

                // If the connection was persistent, flush the read buffer in case there
                // was a previous incomplete request on this connection
                if ( $this->_persistent ) {
                        $this->_flush_read_buffer( $sock );
                }
                return true;
        }

        // }}}
        // {{{ _dead_sock()

        function _dead_sock( $sock ) {
                $host = array_search( $sock, $this->_cache_sock );
                $this->_dead_host( $host );
        }

        function _dead_host( $host ) {
                $parts = explode( ':', $host );
                $ip = $parts[0];
                $this->_host_dead[$ip] = time() + 30 + intval( rand( 0, 10 ) );
                $this->_host_dead[$host] = $this->_host_dead[$ip];
                unset( $this->_cache_sock[$host] );
        }

        // }}}
        // {{{ get_sock()

        function get_sock( $key ) {
                if ( !$this->_active ) {
                        return false;
                }

                if ( $this->_single_sock !== null ) {
                        return $this->sock_to_host( $this->_single_sock );
                }

                $hv = is_array( $key ) ? intval( $key[0] ) : $this->_hashfunc( $key );
                if ( $this->_buckets === null ) {
                        $bu = array();
                        foreach ( $this->_servers as $v ) {
                                if ( is_array( $v ) ) {
                                        for( $i = 0; $i < $v[1]; $i++ ) {
                                                $bu[] = $v[0];
                                        }
                                } else {
                                        $bu[] = $v;
                                }
                        }
                        $this->_buckets = $bu;
                        $this->_bucketcount = count( $bu );
                }

                $realkey = is_array( $key ) ? $key[1] : $key;
                for( $tries = 0; $tries < 20; $tries++ ) {
                        $host = $this->_buckets[$hv % $this->_bucketcount];
                        $sock = $this->sock_to_host( $host );
                        if ( is_resource( $sock ) ) {
                                return $sock;
                        }
                        $hv = $this->_hashfunc( $hv . $realkey );
                }

                return false;
        }

        // }}}
        // {{{ _hashfunc()

        function _hashfunc( $key ) {
                # Hash function must be in [0,0x7ffffff]
                # We take the first 31 bits of the MD5 hash, which unlike the hash
                # function used in a previous version of this client, works
                return hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff;
        }

        // }}}
        // {{{ _incrdecr()

        function _incrdecr( $cmd, $key, $amt = 1 ) {
                if ( !$this->_active ) {
                        return null;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return null;
                }

                $key = is_array( $key ) ? $key[1] : $key;
                if ( isset( $this->stats[$cmd] ) ) {
                        $this->stats[$cmd]++;
                } else {
                        $this->stats[$cmd] = 1;
                }
                if ( !$this->_fwrite( $sock, "$cmd $key $amt\r\n" ) ) {
                        return null;
                }

                $line = $this->_fgets( $sock );
                $match = array();
                if ( !preg_match( '/^(\d+)/', $line, $match ) ) {
                        return null;
                }
                return $match[1];
        }

        // }}}
        // {{{ _load_items()

        function _load_items( $sock, &$ret ) {
                while ( 1 ) {
                        $decl = $this->_fgets( $sock );
                        if( $decl === false ) {
                                return false;
                        } elseif ( $decl == "END" ) {
                                return true;
                        } elseif ( preg_match( '/^VALUE (\S+) (\d+) (\d+)$/', $decl, $match ) ) {
                                list( $rkey, $flags, $len ) = array( $match[1], $match[2], $match[3] );
                                $data = $this->_fread( $sock, $len + 2 );
                                if ( $data === false ) {
                                        return false;
                                }
                                if ( substr( $data, -2 ) !== "\r\n" ) {
                                        $this->_handle_error( $sock, 
                                                'line ending missing from data block from $1' );
                                        return false;
                                }
                                $data = substr( $data, 0, -2 );
                                $ret[$rkey] = $data;

                                if ( $this->_have_zlib && $flags & self::COMPRESSED ) {
                                        $ret[$rkey] = gzuncompress( $ret[$rkey] );
                                }

                                if ( $flags & self::SERIALIZED ) {
                                        $ret[$rkey] = unserialize( $ret[$rkey] );
                                }

                        } else {
                                $this->_handle_error( $sock, 'Error parsing response from $1' );
                                return false;
                        }
                }
        }

        // }}}
        // {{{ _set()

        function _set( $cmd, $key, $val, $exp ) {
                if ( !$this->_active ) {
                        return false;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return false;
                }

                if ( isset( $this->stats[$cmd] ) ) {
                        $this->stats[$cmd]++;
                } else {
                        $this->stats[$cmd] = 1;
                }

                $flags = 0;

                if ( !is_scalar( $val ) ) {
                        $val = serialize( $val );
                        $flags |= self::SERIALIZED;
                        if ( $this->_debug ) {
                                $this->_debugprint( sprintf( "client: serializing data as it is not scalar\n" ) );
                        }
                }

                $len = strlen( $val );

                if ( $this->_have_zlib && $this->_compress_enable &&
                         $this->_compress_threshold && $len >= $this->_compress_threshold )
                {
                        $c_val = gzcompress( $val, 9 );
                        $c_len = strlen( $c_val );

                        if ( $c_len < $len * ( 1 - self::COMPRESSION_SAVINGS ) ) {
                                if ( $this->_debug ) {
                                        $this->_debugprint( sprintf( "client: compressing data; was %d bytes is now %d bytes\n", $len, $c_len ) );
                                }
                                $val = $c_val;
                                $len = $c_len;
                                $flags |= self::COMPRESSED;
                        }
                }
                if ( !$this->_fwrite( $sock, "$cmd $key $flags $exp $len\r\n$val\r\n" ) ) {
                        return false;
                }

                $line = $this->_fgets( $sock );

                if ( $this->_debug ) {
                        $this->_debugprint( sprintf( "%s %s (%s)\n", $cmd, $key, $line ) );
                }
                if ( $line == "STORED" ) {
                        return true;
                }
                return false;
        }

        // }}}
        // {{{ sock_to_host()

        function sock_to_host( $host ) {
                if ( isset( $this->_cache_sock[$host] ) ) {
                        return $this->_cache_sock[$host];
                }

                $sock = null;
                $now = time();
                list( $ip, /* $port */) = explode( ':', $host );
                if ( isset( $this->_host_dead[$host] ) && $this->_host_dead[$host] > $now ||
                        isset( $this->_host_dead[$ip] ) && $this->_host_dead[$ip] > $now
                ) {
                        return null;
                }

                if ( !$this->_connect_sock( $sock, $host ) ) {
                        return null;
                }

                // Do not buffer writes
                stream_set_write_buffer( $sock, 0 );

                $this->_cache_sock[$host] = $sock;

                return $this->_cache_sock[$host];
        }

        function _debugprint( $text ) {
                global $wgDebugLogGroups;
                if( !isset( $wgDebugLogGroups['memcached'] ) ) {
                        # Prefix message since it will end up in main debug log file
                        $text = "memcached: $text";
                }
                wfDebugLog( 'memcached', $text );
        }

        function _error_log( $text ) {
                wfDebugLog( 'memcached-serious', "Memcached error: $text" );
        }

        function _fwrite( $sock, $buf ) {
                $bytesWritten = 0;
                $bufSize = strlen( $buf );
                while ( $bytesWritten < $bufSize  ) {
                        $result = fwrite( $sock, $buf );
                        $data = stream_get_meta_data( $sock );
                        if ( $data['timed_out'] ) {
                                $this->_handle_error( $sock, 'timeout writing to $1' );
                                return false;
                        }
                        // Contrary to the documentation, fwrite() returns zero on error in PHP 5.3.
                        if ( $result === false || $result === 0 ) {
                                $this->_handle_error( $sock, 'error writing to $1' );
                                return false;
                        }
                        $bytesWritten += $result;
                }

                return true;
        }

        function _handle_error( $sock, $msg ) {
                $peer = stream_socket_get_name( $sock, true  );
                if ( strval( $peer ) === '' ) {
                        $peer = array_search( $sock, $this->_cache_sock );
                        if ( $peer === false ) {
                                $peer = '[unknown host]';
                        }
                }
                $msg = str_replace( '$1', $peer, $msg );
                $this->_error_log( "$msg\n" );
                $this->_dead_sock( $sock );
        }

        function _fread( $sock, $len ) {
                $buf = '';
                while ( $len > 0 ) {
                        $result = fread( $sock, $len );
                        $data = stream_get_meta_data( $sock );
                        if ( $data['timed_out'] ) {
                                $this->_handle_error( $sock, 'timeout reading from $1' );
                                return false;
                        }
                        if ( $result === false ) {
                                $this->_handle_error( $sock, 'error reading buffer from $1' );
                                return false;
                        }
                        if ( $result === '' ) {
                                // This will happen if the remote end of the socket is shut down
                                $this->_handle_error( $sock, 'unexpected end of file reading from $1' );
                                return false;
                        }
                        $len -= strlen( $result );
                        $buf .= $result;
                }
                return $buf;
        }

        function _fgets( $sock ) {
                $result = fgets( $sock );
                // fgets() may return a partial line if there is a select timeout after
                // a successful recv(), so we have to check for a timeout even if we 
                // got a string response.
                $data = stream_get_meta_data( $sock );
                if ( $data['timed_out'] ) {
                        $this->_handle_error( $sock, 'timeout reading line from $1' );
                        return false;
                }
                if ( $result === false ) {
                        $this->_handle_error( $sock, 'error reading line from $1' );
                        return false;
                }
                if ( substr( $result, -2 ) === "\r\n" ) {
                        $result = substr( $result, 0, -2 );
                } elseif ( substr( $result, -1 ) === "\n" ) {
                        $result = substr( $result, 0, -1 );
                } else {
                        $this->_handle_error( $sock, 'line ending missing in response from $1' );
                        return false;
                }
                return $result;
        }

        function _flush_read_buffer( $f ) {
                if ( !is_resource( $f ) ) {
                        return;
                }
                $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 );
                while ( $n == 1 && !feof( $f ) ) {
                        fread( $f, 1024 );
                        $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 );
                }
        }

        // }}}
        // }}}
        // }}}
}


// }}}

class MemCachedClientforWiki extends MWMemcached {
}