MemcachedClient.php

Go to the documentation of this file.

<?php
// {{{ requirements
// }}}

// {{{ class MWMemcached
class MWMemcached {
        // {{{ properties
        // {{{ public

        // {{{ constants
        // {{{ flags

        const SERIALIZED = 1;

        const COMPRESSED = 2;

        // }}}

        const COMPRESSION_SAVINGS = 0.20;

        // }}}


        var $stats;

        // }}}
        // {{{ private

        var $_cache_sock;

        var $_debug;

        var $_host_dead;

        var $_have_zlib;

        var $_compress_enable;

        var $_compress_threshold;

        var $_persistent;

        var $_single_sock;

        var $_servers;

        var $_buckets;

        var $_bucketcount;

        var $_active;

        var $_timeout_seconds;

        var $_timeout_microseconds;

        var $_connect_timeout;

        var $_connect_attempts;

        // }}}
        // }}}
        // {{{ methods
        // {{{ public functions
        // {{{ memcached()

        public function __construct( $args ) {
                $this->set_servers( isset( $args['servers'] ) ? $args['servers'] : array() );
                $this->_debug = isset( $args['debug'] ) ? $args['debug'] : false;
                $this->stats = array();
                $this->_compress_threshold = isset( $args['compress_threshold'] ) ? $args['compress_threshold'] : 0;
                $this->_persistent = isset( $args['persistent'] ) ? $args['persistent'] : false;
                $this->_compress_enable = true;
                $this->_have_zlib = function_exists( 'gzcompress' );

                $this->_cache_sock = array();
                $this->_host_dead = array();

                $this->_timeout_seconds = 0;
                $this->_timeout_microseconds = isset( $args['timeout'] ) ? $args['timeout'] : 100000;

                $this->_connect_timeout = isset( $args['connect_timeout'] ) ? $args['connect_timeout'] : 0.1;
                $this->_connect_attempts = 2;
        }

        // }}}
        // {{{ add()

        public function add( $key, $val, $exp = 0 ) {
                return $this->_set( 'add', $key, $val, $exp );
        }

        // }}}
        // {{{ decr()

        public function decr( $key, $amt = 1 ) {
                return $this->_incrdecr( 'decr', $key, $amt );
        }

        // }}}
        // {{{ delete()

        public function delete( $key, $time = 0 ) {
                if ( !$this->_active ) {
                        return false;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return false;
                }

                $key = is_array( $key ) ? $key[1] : $key;

                if ( isset( $this->stats['delete'] ) ) {
                        $this->stats['delete']++;
                } else {
                        $this->stats['delete'] = 1;
                }
                $cmd = "delete $key $time\r\n";
                if( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) {
                        $this->_dead_sock( $sock );
                        return false;
                }
                $res = trim( fgets( $sock ) );

                if ( $this->_debug ) {
                        $this->_debugprint( sprintf( "MemCache: delete %s (%s)\n", $key, $res ) );
                }

                if ( $res == "DELETED" ) {
                        return true;
                }
                return false;
        }

        public function lock( $key, $timeout = 0 ) {
                /* stub */
                return true;
        }

        public function unlock( $key ) {
                /* stub */
                return true;
        }

        // }}}
        // {{{ disconnect_all()

        public function disconnect_all() {
                foreach ( $this->_cache_sock as $sock ) {
                        fclose( $sock );
                }

                $this->_cache_sock = array();
        }

        // }}}
        // {{{ enable_compress()

        public function enable_compress( $enable ) {
                $this->_compress_enable = $enable;
        }

        // }}}
        // {{{ forget_dead_hosts()

        public function forget_dead_hosts() {
                $this->_host_dead = array();
        }

        // }}}
        // {{{ get()

        public function get( $key ) {
                wfProfileIn( __METHOD__ );

                if ( $this->_debug ) {
                        $this->_debugprint( "get($key)\n" );
                }

                if ( !$this->_active ) {
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $sock = $this->get_sock( $key );

                if ( !is_resource( $sock ) ) {
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $key = is_array( $key ) ? $key[1] : $key;
                if ( isset( $this->stats['get'] ) ) {
                        $this->stats['get']++;
                } else {
                        $this->stats['get'] = 1;
                }

                $cmd = "get $key\r\n";
                if ( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) {
                        $this->_dead_sock( $sock );
                        wfProfileOut( __METHOD__ );
                        return false;
                }

                $val = array();
                $this->_load_items( $sock, $val );

                if ( $this->_debug ) {
                        foreach ( $val as $k => $v ) {
                                $this->_debugprint( sprintf( "MemCache: sock %s got %s\n", serialize( $sock ), $k ) );
                        }
                }

                $value = false;
                if ( isset( $val[$key] ) ) {
                        $value = $val[$key];
                }
                wfProfileOut( __METHOD__ );
                return $value;
        }

        // }}}
        // {{{ get_multi()

        public function get_multi( $keys ) {
                if ( !$this->_active ) {
                        return false;
                }

                if ( isset( $this->stats['get_multi'] ) ) {
                        $this->stats['get_multi']++;
                } else {
                        $this->stats['get_multi'] = 1;
                }
                $sock_keys = array();

                foreach ( $keys as $key ) {
                        $sock = $this->get_sock( $key );
                        if ( !is_resource( $sock ) ) {
                                continue;
                        }
                        $key = is_array( $key ) ? $key[1] : $key;
                        if ( !isset( $sock_keys[$sock] ) ) {
                                $sock_keys[$sock] = array();
                                $socks[] = $sock;
                        }
                        $sock_keys[$sock][] = $key;
                }

                // Send out the requests
                foreach ( $socks as $sock ) {
                        $cmd = 'get';
                        foreach ( $sock_keys[$sock] as $key ) {
                                $cmd .= ' ' . $key;
                        }
                        $cmd .= "\r\n";

                        if ( $this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) {
                                $gather[] = $sock;
                        } else {
                                $this->_dead_sock( $sock );
                        }
                }

                // Parse responses
                $val = array();
                foreach ( $gather as $sock ) {
                        $this->_load_items( $sock, $val );
                }

                if ( $this->_debug ) {
                        foreach ( $val as $k => $v ) {
                                $this->_debugprint( sprintf( "MemCache: got %s\n", $k ) );
                        }
                }

                return $val;
        }

        // }}}
        // {{{ incr()

        public function incr( $key, $amt = 1 ) {
                return $this->_incrdecr( 'incr', $key, $amt );
        }

        // }}}
        // {{{ replace()

        public function replace( $key, $value, $exp = 0 ) {
                return $this->_set( 'replace', $key, $value, $exp );
        }

        // }}}
        // {{{ run_command()

        public function run_command( $sock, $cmd ) {
                if ( !is_resource( $sock ) ) {
                        return array();
                }

                if ( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) {
                        return array();
                }

                while ( true ) {
                        $res = fgets( $sock );
                        $ret[] = $res;
                        if ( preg_match( '/^END/', $res ) ) {
                                break;
                        }
                        if ( strlen( $res ) == 0 ) {
                                break;
                        }
                }
                return $ret;
        }

        // }}}
        // {{{ set()

        public function set( $key, $value, $exp = 0 ) {
                return $this->_set( 'set', $key, $value, $exp );
        }

        // }}}
        // {{{ set_compress_threshold()

        public function set_compress_threshold( $thresh ) {
                $this->_compress_threshold = $thresh;
        }

        // }}}
        // {{{ set_debug()

        public function set_debug( $dbg ) {
                $this->_debug = $dbg;
        }

        // }}}
        // {{{ set_servers()

        public function set_servers( $list ) {
                $this->_servers = $list;
                $this->_active = count( $list );
                $this->_buckets = null;
                $this->_bucketcount = 0;

                $this->_single_sock = null;
                if ( $this->_active == 1 ) {
                        $this->_single_sock = $this->_servers[0];
                }
        }

        public function set_timeout( $seconds, $microseconds ) {
                $this->_timeout_seconds = $seconds;
                $this->_timeout_microseconds = $microseconds;
        }

        // }}}
        // }}}
        // {{{ private methods
        // {{{ _close_sock()

        function _close_sock( $sock ) {
                $host = array_search( $sock, $this->_cache_sock );
                fclose( $this->_cache_sock[$host] );
                unset( $this->_cache_sock[$host] );
        }

        // }}}
        // {{{ _connect_sock()

        function _connect_sock( &$sock, $host ) {
                list( $ip, $port ) = explode( ':', $host );
                $sock = false;
                $timeout = $this->_connect_timeout;
                $errno = $errstr = null;
                for( $i = 0; !$sock && $i < $this->_connect_attempts; $i++ ) {
                        wfSuppressWarnings();
                        if ( $this->_persistent == 1 ) {
                                $sock = pfsockopen( $ip, $port, $errno, $errstr, $timeout );
                        } else {
                                $sock = fsockopen( $ip, $port, $errno, $errstr, $timeout );
                        }
                        wfRestoreWarnings();
                }
                if ( !$sock ) {
                        if ( $this->_debug ) {
                                $this->_debugprint( "Error connecting to $host: $errstr\n" );
                        }
                        return false;
                }

                // Initialise timeout
                stream_set_timeout( $sock, $this->_timeout_seconds, $this->_timeout_microseconds );

                return true;
        }

        // }}}
        // {{{ _dead_sock()

        function _dead_sock( $sock ) {
                $host = array_search( $sock, $this->_cache_sock );
                $this->_dead_host( $host );
        }

        function _dead_host( $host ) {
                $parts = explode( ':', $host );
                $ip = $parts[0];
                $this->_host_dead[$ip] = time() + 30 + intval( rand( 0, 10 ) );
                $this->_host_dead[$host] = $this->_host_dead[$ip];
                unset( $this->_cache_sock[$host] );
        }

        // }}}
        // {{{ get_sock()

        function get_sock( $key ) {
                if ( !$this->_active ) {
                        return false;
                }

                if ( $this->_single_sock !== null ) {
                        $this->_flush_read_buffer( $this->_single_sock );
                        return $this->sock_to_host( $this->_single_sock );
                }

                $hv = is_array( $key ) ? intval( $key[0] ) : $this->_hashfunc( $key );

                if ( $this->_buckets === null ) {
                        foreach ( $this->_servers as $v ) {
                                if ( is_array( $v ) ) {
                                        for( $i = 0; $i < $v[1]; $i++ ) {
                                                $bu[] = $v[0];
                                        }
                                } else {
                                        $bu[] = $v;
                                }
                        }
                        $this->_buckets = $bu;
                        $this->_bucketcount = count( $bu );
                }

                $realkey = is_array( $key ) ? $key[1] : $key;
                for( $tries = 0; $tries < 20; $tries++ ) {
                        $host = $this->_buckets[$hv % $this->_bucketcount];
                        $sock = $this->sock_to_host( $host );
                        if ( is_resource( $sock ) ) {
                                $this->_flush_read_buffer( $sock );
                                return $sock;
                        }
                        $hv = $this->_hashfunc( $hv . $realkey );
                }

                return false;
        }

        // }}}
        // {{{ _hashfunc()

        function _hashfunc( $key ) {
                # Hash function must on [0,0x7ffffff]
                # We take the first 31 bits of the MD5 hash, which unlike the hash
                # function used in a previous version of this client, works
                return hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff;
        }

        // }}}
        // {{{ _incrdecr()

        function _incrdecr( $cmd, $key, $amt = 1 ) {
                if ( !$this->_active ) {
                        return null;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return null;
                }

                $key = is_array( $key ) ? $key[1] : $key;
                if ( isset( $this->stats[$cmd] ) ) {
                        $this->stats[$cmd]++;
                } else {
                        $this->stats[$cmd] = 1;
                }
                if ( !$this->_safe_fwrite( $sock, "$cmd $key $amt\r\n" ) ) {
                        return $this->_dead_sock( $sock );
                }

                $line = fgets( $sock );
                $match = array();
                if ( !preg_match( '/^(\d+)/', $line, $match ) ) {
                        return null;
                }
                return $match[1];
        }

        // }}}
        // {{{ _load_items()

        function _load_items( $sock, &$ret ) {
                while ( 1 ) {
                        $decl = fgets( $sock );
                        if ( $decl == "END\r\n" ) {
                                return true;
                        } elseif ( preg_match( '/^VALUE (\S+) (\d+) (\d+)\r\n$/', $decl, $match ) ) {
                                list( $rkey, $flags, $len ) = array( $match[1], $match[2], $match[3] );
                                $bneed = $len + 2;
                                $offset = 0;

                                while ( $bneed > 0 ) {
                                        $data = fread( $sock, $bneed );
                                        $n = strlen( $data );
                                        if ( $n == 0 ) {
                                                break;
                                        }
                                        $offset += $n;
                                        $bneed -= $n;
                                        if ( isset( $ret[$rkey] ) ) {
                                                $ret[$rkey] .= $data;
                                        } else {
                                                $ret[$rkey] = $data;
                                        }
                                }

                                if ( $offset != $len + 2 ) {
                                        // Something is borked!
                                        if ( $this->_debug ) {
                                                $this->_debugprint( sprintf( "Something is borked!  key %s expecting %d got %d length\n", $rkey, $len + 2, $offset ) );
                                        }

                                        unset( $ret[$rkey] );
                                        $this->_close_sock( $sock );
                                        return false;
                                }

                                if ( $this->_have_zlib && $flags & self::COMPRESSED ) {
                                        $ret[$rkey] = gzuncompress( $ret[$rkey] );
                                }

                                $ret[$rkey] = rtrim( $ret[$rkey] );

                                if ( $flags & self::SERIALIZED ) {
                                        $ret[$rkey] = unserialize( $ret[$rkey] );
                                }

                        } else {
                                $this->_debugprint( "Error parsing memcached response\n" );
                                return 0;
                        }
                }
        }

        // }}}
        // {{{ _set()

        function _set( $cmd, $key, $val, $exp ) {
                if ( !$this->_active ) {
                        return false;
                }

                $sock = $this->get_sock( $key );
                if ( !is_resource( $sock ) ) {
                        return false;
                }

                if ( isset( $this->stats[$cmd] ) ) {
                        $this->stats[$cmd]++;
                } else {
                        $this->stats[$cmd] = 1;
                }

                // TTLs higher than 30 days will be detected as absolute TTLs
                // (UNIX timestamps), and will result in the cache entry being
                // discarded immediately because the expiry is in the past.
                // Clamp expiries >30d at 30d, unless they're >=1e9 in which
                // case they are likely to really be absolute (1e9 = 2011-09-09)
                if ( $exp > 2592000 && $exp < 1000000000 ) {
                        $exp = 2592000;
                }

                $flags = 0;

                if ( !is_scalar( $val ) ) {
                        $val = serialize( $val );
                        $flags |= self::SERIALIZED;
                        if ( $this->_debug ) {
                                $this->_debugprint( sprintf( "client: serializing data as it is not scalar\n" ) );
                        }
                }

                $len = strlen( $val );

                if ( $this->_have_zlib && $this->_compress_enable &&
                         $this->_compress_threshold && $len >= $this->_compress_threshold )
                {
                        $c_val = gzcompress( $val, 9 );
                        $c_len = strlen( $c_val );

                        if ( $c_len < $len * ( 1 - self::COMPRESSION_SAVINGS ) ) {
                                if ( $this->_debug ) {
                                        $this->_debugprint( sprintf( "client: compressing data; was %d bytes is now %d bytes\n", $len, $c_len ) );
                                }
                                $val = $c_val;
                                $len = $c_len;
                                $flags |= self::COMPRESSED;
                        }
                }
                if ( !$this->_safe_fwrite( $sock, "$cmd $key $flags $exp $len\r\n$val\r\n" ) ) {
                        return $this->_dead_sock( $sock );
                }

                $line = trim( fgets( $sock ) );

                if ( $this->_debug ) {
                        $this->_debugprint( sprintf( "%s %s (%s)\n", $cmd, $key, $line ) );
                }
                if ( $line == "STORED" ) {
                        return true;
                }
                return false;
        }

        // }}}
        // {{{ sock_to_host()

        function sock_to_host( $host ) {
                if ( isset( $this->_cache_sock[$host] ) ) {
                        return $this->_cache_sock[$host];
                }

                $sock = null;
                $now = time();
                list( $ip, /* $port */) = explode( ':', $host );
                if ( isset( $this->_host_dead[$host] ) && $this->_host_dead[$host] > $now ||
                        isset( $this->_host_dead[$ip] ) && $this->_host_dead[$ip] > $now
                ) {
                        return null;
                }

                if ( !$this->_connect_sock( $sock, $host ) ) {
                        return $this->_dead_host( $host );
                }

                // Do not buffer writes
                stream_set_write_buffer( $sock, 0 );

                $this->_cache_sock[$host] = $sock;

                return $this->_cache_sock[$host];
        }

        function _debugprint( $str ) {
                print( $str );
        }

        /*
        function _safe_fwrite( $f, $buf, $len = false ) {
                stream_set_blocking( $f, 0 );

                if ( $len === false ) {
                        wfDebug( "Writing " . strlen( $buf ) . " bytes\n" );
                        $bytesWritten = fwrite( $f, $buf );
                } else {
                        wfDebug( "Writing $len bytes\n" );
                        $bytesWritten = fwrite( $f, $buf, $len );
                }
                $n = stream_select( $r = null, $w = array( $f ), $e = null, 10, 0 );
                #   $this->_timeout_seconds, $this->_timeout_microseconds );

                wfDebug( "stream_select returned $n\n" );
                stream_set_blocking( $f, 1 );
                return $n == 1;
                return $bytesWritten;
        }*/

        function _safe_fwrite( $f, $buf, $len = false ) {
                if ( $len === false ) {
                        $bytesWritten = fwrite( $f, $buf );
                } else {
                        $bytesWritten = fwrite( $f, $buf, $len );
                }
                return $bytesWritten;
        }

        function _flush_read_buffer( $f ) {
                if ( !is_resource( $f ) ) {
                        return;
                }
                $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 );
                while ( $n == 1 && !feof( $f ) ) {
                        fread( $f, 1024 );
                        $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 );
                }
        }

        // }}}
        // }}}
        // }}}
}

// vim: sts=3 sw=3 et

// }}}

class MemCachedClientforWiki extends MWMemcached {
        function _debugprint( $text ) {
                wfDebug( "memcached: $text" );
        }
}