diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/ChangeLog memcached-1.1.13-pre/ChangeLog --- memcached-1.1.12/ChangeLog 2005-04-04 17:16:37.000000000 -0700 +++ memcached-1.1.13-pre/ChangeLog 2006-04-30 14:29:20.000000000 -0700 @@ -1,3 +1,30 @@ +2006-04-30 + * River Tarnell: autoconf work for Solaris 10. Brad: + merge and verify it works on Nexenta. + +2006-03-04 + * avva: bucket/generation patch (old, but Brad's just finally + committing it) + +2006-01-01 + * Brad Fitzpatrick : allocate 1 slab per class + on start-up, to avoid confusing users with out-of-memory errors + later. this is 18 MB of allocation on start, unless max memory + allowed with -m is lower, in which case only the smaller slab + classes are allocated. + +2005-08-09 + * Elizabeth Mattijsen : needed a way to flush all + memcached backend servers, but not at exactly the same time (to + reduce load peaks), I've added some simple functionality to the + memcached protocol in the "flush_all" command that allows you to + specify a time at which the flush will actually occur (instead of + always at the moment the "flush_all" command is received). + +2005-05-25 + * patch from Peter van Dijk to make + stderr unbuffered, for running under daemontools + 2005-04-04 * patch from Don MacAskill 'flush_all' doesn't seem to work properly. Basically, if you try to add a key which diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/Makefile.am memcached-1.1.13-pre/Makefile.am --- memcached-1.1.12/Makefile.am 2005-01-14 13:59:59.000000000 -0800 +++ memcached-1.1.13-pre/Makefile.am 2006-04-30 14:22:53.000000000 -0700 @@ -1,6 +1,7 @@ bin_PROGRAMS = memcached memcached_SOURCES = memcached.c slabs.c items.c memcached.h assoc.c +memcached_LDADD = @LIBOBJS@ SUBDIRS = doc DIST_DIRS = scripts diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/assoc.c memcached-1.1.13-pre/assoc.c --- memcached-1.1.12/assoc.c 2005-04-04 17:14:55.000000000 -0700 +++ memcached-1.1.13-pre/assoc.c 2006-04-30 14:22:37.000000000 -0700 @@ -10,7 +10,7 @@ * * The rest of the file is licensed under the BSD license. See LICENSE. * - * $Id: assoc.c,v 1.6 2003/08/11 05:44:26 bradfitz Exp $ + * $Id: assoc.c 257 2006-03-23 06:41:44Z bradfitz $ */ #include diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/configure.ac memcached-1.1.13-pre/configure.ac --- memcached-1.1.12/configure.ac 2005-04-04 17:15:12.000000000 -0700 +++ memcached-1.1.13-pre/configure.ac 2006-04-30 14:33:11.000000000 -0700 @@ -1,5 +1,5 @@ AC_PREREQ(2.52) -AC_INIT(memcached, 1.1.12, brad@danga.com) +AC_INIT(memcached, 1.1.13-pre, brad@danga.com) AC_CANONICAL_SYSTEM AC_CONFIG_SRCDIR(memcached.c) AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION) @@ -19,6 +19,13 @@ AC_CHECK_LIB(event, event_set, , [AC_MSG_ERROR(libevent is required. You can get it from $LIBEVENT_URL)]) +AC_SEARCH_LIBS(socket, socket) +AC_SEARCH_LIBS(gethostbyname, nsl) +AC_SEARCH_LIBS(mallinfo, malloc) + +AC_CHECK_FUNC(daemon,AC_DEFINE([HAVE_DAEMON],,[Define this if you have daemon()]),[AC_LIBOBJ(daemon)]) + + AC_CHECK_HEADER(malloc.h, AC_DEFINE(HAVE_MALLOC_H,,[do we have malloc.h?])) AC_CHECK_MEMBER([struct mallinfo.arena], [ AC_DEFINE(HAVE_STRUCT_MALLINFO,,[do we have stuct mallinfo?]) diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/daemon.c memcached-1.1.13-pre/daemon.c --- memcached-1.1.12/daemon.c 1969-12-31 16:00:00.000000000 -0800 +++ memcached-1.1.13-pre/daemon.c 2006-04-30 13:13:07.000000000 -0700 @@ -0,0 +1,71 @@ +/* $Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $ */ +/* $NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $ */ + +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined __SUNPRO_C || defined __DECC || defined __HP_cc +# pragma ident "@(#)$Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $" +# pragma ident "$NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $" +#endif + +#include +#include +#include + +int +daemon(nochdir, noclose) + int nochdir, noclose; +{ + int fd; + + switch (fork()) { + case -1: + return (-1); + case 0: + break; + default: + _exit(0); + } + + if (setsid() == -1) + return (-1); + + if (!nochdir) + (void)chdir("/"); + + if (!noclose && (fd = open("/dev/null", O_RDWR, 0)) != -1) { + (void)dup2(fd, STDIN_FILENO); + (void)dup2(fd, STDOUT_FILENO); + (void)dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) + (void)close(fd); + } + return (0); +} diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/doc/OSX.txt memcached-1.1.13-pre/doc/OSX.txt --- memcached-1.1.12/doc/OSX.txt 1969-12-31 16:00:00.000000000 -0800 +++ memcached-1.1.13-pre/doc/OSX.txt 2006-04-30 14:22:37.000000000 -0700 @@ -0,0 +1,28 @@ +memcached is slow on OSX because: + + -- OSX's kqueue is broken + -- OSX's TCP_NOPUSH stuff is different/broken + +So there are reports that this works and make memcached fast on OS X: + + Two simple changes: + + First, in memcached.c (in the memcached source directory) add + (anywhere above line 105, which reads #ifdef TCP_NOPUSH) the line: + + #undef TCP_NOPUSH + + I just added it on the line above the #ifdef line. + + Rebuild memcached (just do a make && sudo make install, don.t need + to re-run configure if you.ve already done it) + + then, set the environment variable EVENT_NOKQUEUE to 1 + + in csh and derivatives: setenv EVENT_NOKQUEUE 1 + + in sh and derivatives (like bash): export EVENT_NOKQUEUE=1 + + then start memcached, and it should be fast (it certainly made a + difference here) + diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/doc/memcached.1 memcached-1.1.13-pre/doc/memcached.1 --- memcached-1.1.12/doc/memcached.1 2004-12-10 16:58:21.000000000 -0800 +++ memcached-1.1.13-pre/doc/memcached.1 2006-04-30 14:22:37.000000000 -0700 @@ -1,4 +1,4 @@ -.TH MEMCACHED 1 "November 12, 2003" +.TH MEMCACHED 1 "April 11, 2005" .SH NAME memcached \- high-performance memory object caching system .SH SYNOPSIS @@ -34,10 +34,6 @@ .B \-m Use MB memory max to use for object storage; the default is 64 megabytes. .TP -.B \-M -Instead of throwing items from the cache when max memory is reached, throw an -error -.TP .B \-c Use max simultaneous connections; the default is 1024. .TP @@ -49,9 +45,6 @@ .B \-p Listen on port , the default is port 11211. .TP -.B \-r -Maximize core file limit -.TP .B \-M Disable automatic removal of items from the cache when out of memory. Additions will not be possible until adequate space is freed up. @@ -71,6 +64,9 @@ .TP .B \-i Print memcached and libevent licenses. +.TP +.B \-P +Print pidfile to , only used under -d option. .br .SH LICENSE The memcached daemon is copyright Danga Interactive and is distributed under diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/doc/protocol.txt memcached-1.1.13-pre/doc/protocol.txt --- memcached-1.1.12/doc/protocol.txt 2004-04-26 14:26:48.000000000 -0700 +++ memcached-1.1.13-pre/doc/protocol.txt 2006-04-30 14:22:37.000000000 -0700 @@ -359,16 +359,17 @@ Other commands -------------- -"flush_all" is a command with no arguments. It always succeeds, -and the server sends "OK\r\n" in response. Its effect is to immediately -invalidate all existing items: none of them will be returned in -response to a retrieval command (unless it's stored again under the -same key *after* flush_all has been executed). flush_all doesn't +"flush_all" is a command with an optional numeric argument. It always +succeeds, and the server sends "OK\r\n" in response. Its effect is to +invalidate all existing items immediately (by default) or after the +expiration specified. After invalidation none of the items will be returned +in response to a retrieval command (unless it's stored again under the +same key *after* flush_all has invalidated the items). flush_all doesn't actually free all the memory taken up by existing items; that will happen gradually as new items are stored. The most precise definition of what flush_all does is the following: it causes all items whose -update time is earlier than the time at which flush_all was executed -to be ignored for retrieval purposes. +update time is earlier than the time at which flush_all was set to be +executed to be ignored for retrieval purposes. "version" is a command with no arguments: diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/items.c memcached-1.1.13-pre/items.c --- memcached-1.1.12/items.c 2004-09-14 10:45:48.000000000 -0700 +++ memcached-1.1.13-pre/items.c 2006-04-30 14:22:37.000000000 -0700 @@ -1,5 +1,5 @@ /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* $Id: items.c,v 1.23 2004/09/13 22:31:53 avva Exp $ */ +/* $Id: items.c 257 2006-03-23 06:41:44Z bradfitz $ */ #include #include diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/memcached.c memcached-1.1.13-pre/memcached.c --- memcached-1.1.12/memcached.c 2005-04-04 17:10:26.000000000 -0700 +++ memcached-1.1.13-pre/memcached.c 2006-04-30 14:22:37.000000000 -0700 @@ -13,7 +13,7 @@ * Anatoly Vorobey * Brad Fitzpatrick * - * $Id: memcached.c,v 1.56 2005/04/05 00:10:26 bradfitz Exp $ + * $Id: memcached.c 260 2006-04-08 19:32:01Z bradfitz $ */ #include "config.h" @@ -56,6 +56,8 @@ static int delcurr; static int deltotal; +int *buckets = 0; /* bucket->generation array for a managed instance */ + time_t realtime(time_t exptime) { time_t now; @@ -93,6 +95,7 @@ settings.verbose = 0; settings.oldest_live = 0; settings.evict_to_free = 1; /* push old items out of cache when memory runs out */ + settings.managed = 0; } conn **freeconns; @@ -141,6 +144,7 @@ return 0; } c->rsize = c->wsize = DATA_BUFFER_SIZE; + c->rcurr = c->rbuf; c->isize = 200; stats.conn_structs++; } @@ -157,7 +161,7 @@ c->rlbytes = 0; c->rbytes = c->wbytes = 0; c->wcurr = c->wbuf; - c->rcurr = c->rbuf; + c->ritem = 0; c->icurr = c->ilist; c->ileft = 0; c->iptr = c->ibuf; @@ -166,6 +170,8 @@ c->write_and_go = conn_read; c->write_and_free = 0; c->item = 0; + c->bucket = -1; + c->gen = 0; c->is_corked = 0; @@ -522,6 +528,20 @@ out_string(c, "CLIENT_ERROR bad command line format"); return; } + + if (settings.managed) { + int bucket = c->bucket; + if (bucket == -1) { + out_string(c, "CLIENT_ERROR no BG data in managed mode"); + return; + } + c->bucket = -1; + if (buckets[bucket] != c->gen) { + out_string(c, "ERROR_NOT_OWNER"); + return; + } + } + expire = realtime(expire); it = item_alloc(key, flags, expire, len+2); if (it == 0) { @@ -534,7 +554,7 @@ c->item_comm = comm; c->item = it; - c->rcurr = ITEM_data(it); + c->ritem = ITEM_data(it); c->rlbytes = it->nbytes; c->state = conn_nread; return; @@ -556,7 +576,20 @@ out_string(c, "CLIENT_ERROR bad command line format"); return; } - + + if (settings.managed) { + int bucket = c->bucket; + if (bucket == -1) { + out_string(c, "CLIENT_ERROR no BG data in managed mode"); + return; + } + c->bucket = -1; + if (buckets[bucket] != c->gen) { + out_string(c, "ERROR_NOT_OWNER"); + return; + } + } + it = assoc_find(key); if (it && (it->it_flags & ITEM_DELETED)) { it = 0; @@ -603,6 +636,10 @@ return; } + if (strncmp(command, "bget ", 5) == 0) { + c->binary = 1; + goto get; + } if (strncmp(command, "get ", 4) == 0) { char *start = command + 4; @@ -611,6 +648,20 @@ int i = 0; item *it; time_t now = time(0); + get: + + if (settings.managed) { + int bucket = c->bucket; + if (bucket == -1) { + out_string(c, "CLIENT_ERROR no BG data in managed mode"); + return; + } + c->bucket = -1; + if (buckets[bucket] != c->gen) { + out_string(c, "ERROR_NOT_OWNER"); + return; + } + } while(sscanf(start, " %250s%n", key, &next) >= 1) { start+=next; @@ -619,8 +670,8 @@ if (it && (it->it_flags & ITEM_DELETED)) { it = 0; } - if (settings.oldest_live && it && - it->time <= settings.oldest_live) { + if (settings.oldest_live && settings.oldest_live <= now && + it && it->time <= settings.oldest_live) { item_unlink(it); it = 0; } @@ -663,6 +714,19 @@ int res; time_t exptime = 0; + if (settings.managed) { + int bucket = c->bucket; + if (bucket == -1) { + out_string(c, "CLIENT_ERROR no BG data in managed mode"); + return; + } + c->bucket = -1; + if (buckets[bucket] != c->gen) { + out_string(c, "ERROR_NOT_OWNER"); + return; + } + } + res = sscanf(command, "%*s %250s %ld", key, &exptime); it = assoc_find(key); if (!it) { @@ -701,14 +765,97 @@ out_string(c, "DELETED"); return; } - + + if (strncmp(command, "own ", 4) == 0) { + int bucket, gen; + char *start = command+4; + if (!settings.managed) { + out_string(c, "CLIENT_ERROR not a managed instance"); + return; + } + if (sscanf(start, "%u:%u\r\n", &bucket,&gen) == 2) { + if ((bucket < 0) || (bucket >= MAX_BUCKETS)) { + out_string(c, "CLIENT_ERROR bucket number out of range"); + return; + } + buckets[bucket] = gen; + out_string(c, "OWNED"); + return; + } else { + out_string(c, "CLIENT_ERROR bad format"); + return; + } + } + + if (strncmp(command, "disown ", 7) == 0) { + int bucket; + char *start = command+7; + if (!settings.managed) { + out_string(c, "CLIENT_ERROR not a managed instance"); + return; + } + if (sscanf(start, "%u\r\n", &bucket) == 1) { + if ((bucket < 0) || (bucket >= MAX_BUCKETS)) { + out_string(c, "CLIENT_ERROR bucket number out of range"); + return; + } + buckets[bucket] = 0; + out_string(c, "DISOWNED"); + return; + } else { + out_string(c, "CLIENT_ERROR bad format"); + return; + } + } + + if (strncmp(command, "bg ", 3) == 0) { + int bucket, gen; + char *start = command+3; + if (!settings.managed) { + out_string(c, "CLIENT_ERROR not a managed instance"); + return; + } + if (sscanf(start, "%u:%u\r\n", &bucket,&gen) == 2) { + /* we never write anything back, even if input's wrong */ + if ((bucket < 0) || (bucket >= MAX_BUCKETS) || (gen<=0)) { + /* do nothing, bad input */ + } else { + c->bucket = bucket; + c->gen = gen; + } + c->state = conn_read; + /* normally conn_write uncorks the connection, but this + is the only time we accept a command w/o writing anything */ + set_cork(c,0); + return; + } else { + out_string(c, "CLIENT_ERROR bad format"); + return; + } + } + if (strncmp(command, "stats", 5) == 0) { process_stat(c, command); return; } - if (strcmp(command, "flush_all") == 0) { - settings.oldest_live = time(0); + if (strncmp(command, "flush_all", 9) == 0) { + time_t exptime = 0; + int res; + + if (strcmp(command, "flush_all") == 0) { + settings.oldest_live = time(0); + out_string(c, "OK"); + return; + } + + res = sscanf(command, "%*s %ld", &exptime); + if (res != 1) { + out_string(c, "ERROR"); + return; + } + + settings.oldest_live = realtime(exptime); out_string(c, "OK"); return; } @@ -750,40 +897,47 @@ } /* - * if we have a complete line in the buffer, process it and move whatever - * remains in the buffer to its beginning. + * if we have a complete line in the buffer, process it. */ int try_read_command(conn *c) { char *el, *cont; if (!c->rbytes) return 0; - el = memchr(c->rbuf, '\n', c->rbytes); + el = memchr(c->rcurr, '\n', c->rbytes); if (!el) return 0; cont = el + 1; - if (el - c->rbuf > 1 && *(el - 1) == '\r') { + if (el - c->rcurr > 1 && *(el - 1) == '\r') { el--; } *el = '\0'; - process_command(c, c->rbuf); + process_command(c, c->rcurr); + + c->rbytes -= (cont - c->rcurr); + c->rcurr = cont; - if (cont - c->rbuf < c->rbytes) { /* more stuff in the buffer */ - memmove(c->rbuf, cont, c->rbytes - (cont - c->rbuf)); - } - c->rbytes -= (cont - c->rbuf); return 1; } /* * read from network as much as we can, handle buffer overflow and connection * close. + * before reading, move the remaining incomplete fragment of a command + * (if any) to the beginning of the buffer. * return 0 if there's nothing to read on the first read. */ int try_read_network(conn *c) { int gotdata = 0; int res; + + if (c->rcurr != c->rbuf) { + if (c->rbytes != 0) /* otherwise there's nothing to copy */ + memmove(c->rbuf, c->rcurr, c->rbytes); + c->rcurr = c->rbuf; + } + while (1) { if (c->rbytes >= c->rsize) { char *new_rbuf = realloc(c->rbuf, c->rsize*2); @@ -795,7 +949,8 @@ c->write_and_go = conn_closing; return 1; } - c->rbuf = new_rbuf; c->rsize *= 2; + c->rcurr = c->rbuf = new_rbuf; + c->rsize *= 2; } res = read(c->sfd, c->rbuf + c->rbytes, c->rsize - c->rbytes); if (res > 0) { @@ -884,7 +1039,7 @@ break; case conn_nread: - /* we are reading rlbytes into rcurr; */ + /* we are reading rlbytes into ritem; */ if (c->rlbytes == 0) { complete_nread(c); break; @@ -892,21 +1047,19 @@ /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes; - memcpy(c->rcurr, c->rbuf, tocopy); - c->rcurr += tocopy; + memcpy(c->ritem, c->rcurr, tocopy); + c->ritem += tocopy; c->rlbytes -= tocopy; - if (c->rbytes > tocopy) { - memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy); - } + c->rcurr += tocopy; c->rbytes -= tocopy; break; } /* now try reading from the socket */ - res = read(c->sfd, c->rcurr, c->rlbytes); + res = read(c->sfd, c->ritem, c->rlbytes); if (res > 0) { stats.bytes_read += res; - c->rcurr += res; + c->ritem += res; c->rlbytes -= res; break; } @@ -941,9 +1094,7 @@ if (c->rbytes > 0) { int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes; c->sbytes -= tocopy; - if (c->rbytes > tocopy) { - memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy); - } + c->rcurr += tocopy; c->rbytes -= tocopy; break; } @@ -1052,7 +1203,11 @@ it = *(c->icurr); assert((it->it_flags & ITEM_SLABBED) == 0); c->iptr = ITEM_data(it); + if (c->binary) { + c->ibytes = it->nbytes - 2; + } else { c->ibytes = it->nbytes; + } c->ipart = 2; break; case 2: @@ -1069,14 +1224,39 @@ case 0: it = *(c->icurr); assert((it->it_flags & ITEM_SLABBED) == 0); + // add branch for binary mode + if (c->binary) { + c->ibuf[0] = 0x01; // key + long int key_size = htonl(it->nkey - 3); + long int value_size = htonl(it->nbytes - 2); + /* XXX NOT SAFE ON 64 BIT FOR PROTOCOL REASONS + NEED TO RECODE TO USE 4 byte INTS ONLY */ + memcpy(c->ibuf + 1, (char *)&key_size, sizeof(key_size)); + memcpy(c->ibuf + 1 + 4, ITEM_key(it), it->nkey - 3); + memcpy(c->ibuf + 1 + 4 + it->nkey - 3, (char *)&value_size, sizeof(value_size)); + c->ibytes = 1 + 4 + it->nkey - 3 + 4; + } else { c->ibytes = sprintf(c->ibuf, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); + } if (settings.verbose > 1) fprintf(stderr, ">%d sending key %s\n", c->sfd, ITEM_key(it)); c->iptr = c->ibuf; c->ipart = 1; break; case 3: + // add branch for binary mode + if (c->binary) { + c->binary = 0; + + c->wbuf[0] = 0x01; + c->wbytes = 1; + c->wcurr = c->wbuf; + c->state = conn_write; + c->write_and_go = conn_read; + + } else { out_string(c, "END"); + } break; } } @@ -1233,6 +1413,7 @@ printf("-vv very verbose (also print client commands/reponses)\n"); printf("-h print this help and exit\n"); printf("-i print memcached and libevent license\n"); + printf("-b run a managed instanced (mnemonic: buckets)\n"); printf("-P save PID in , only used with -d option\n"); return; } @@ -1353,9 +1534,15 @@ /* init settings */ settings_init(); + /* set stderr non-buffering (for running under, say, daemontools) */ + setbuf(stderr, NULL); + /* process arguments */ - while ((c = getopt(argc, argv, "p:m:Mc:khirvdl:u:P:")) != -1) { + while ((c = getopt(argc, argv, "bp:m:Mc:khirvdl:u:P:")) != -1) { switch (c) { + case 'b': + settings.managed = 1; + break; case 'p': settings.port = atoi(optarg); break; @@ -1503,6 +1690,16 @@ conn_init(); slabs_init(settings.maxbytes); + /* managed instance? alloc and zero a bucket array */ + if (settings.managed) { + buckets = malloc(sizeof(int)*MAX_BUCKETS); + if (buckets == 0) { + fprintf(stderr, "failed to allocate the bucket array"); + exit(1); + } + memset(buckets, 0, sizeof(int)*MAX_BUCKETS); + } + /* lock paged memory if needed */ if (lock_memory) { #ifdef HAVE_MLOCKALL diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/memcached.h memcached-1.1.13-pre/memcached.h --- memcached-1.1.12/memcached.h 2005-04-04 17:14:55.000000000 -0700 +++ memcached-1.1.13-pre/memcached.h 2006-04-30 14:22:37.000000000 -0700 @@ -1,5 +1,5 @@ /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* $Id: memcached.h,v 1.21 2004/02/24 23:42:02 bradfitz Exp $ */ +/* $Id: memcached.h 258 2006-03-29 11:32:40Z sky $ */ #define DATA_BUFFER_SIZE 2048 @@ -29,6 +29,7 @@ int port; struct in_addr interface; int verbose; + int managed; /* if 1, a tracker manages virtual buckets */ time_t oldest_live; /* ignore existing items older than this */ int evict_to_free; }; @@ -83,11 +84,12 @@ int state; struct event event; short ev_flags; - short which; /* which events were just triggered */ + short which; /* which events were just triggered */ - char *rbuf; - int rsize; - int rbytes; + char *rbuf; /* buffer to read commands into */ + char *rcurr; /* but if we parsed some already, this is where we stopped */ + int rsize; /* total allocated size of rbuf */ + int rbytes; /* how much data, starting from rcur, do we have unparsed */ char *wbuf; char *wcurr; @@ -97,7 +99,7 @@ void *write_and_free; /* free this memory after finishing writing */ char is_corked; /* boolean, connection is corked */ - char *rcurr; + char *ritem; /* when we read in an item's value, it goes here */ int rlbytes; /* data for the nread state */ @@ -123,9 +125,16 @@ char ibuf[300]; /* for VALUE lines */ char *iptr; int ibytes; + int binary; /* are we in binary mode */ + int bucket; /* bucket number for the next command, if running as + a managed instance. -1 (_not_ 0) means invalid. */ + int gen; /* generation requested for the bucket */ } conn; +/* number of virtual buckets for a managed instance */ +#define MAX_BUCKETS 32768 + /* listening socket */ extern int l_socket; @@ -150,6 +159,14 @@ /* Init the subsystem. The argument is the limit on no. of bytes to allocate, 0 if no limit */ void slabs_init(unsigned int limit); +/* Preallocate as many slab pages as possible (called from slabs_init) + on start-up, so users don't get confused out-of-memory errors when + they do have free (in-slab) space, but no space to make new slabs. + if maxslabs is 18 (POWER_LARGEST - POWER_SMALLEST + 1), then all + slab types can be made. if max memory is less than 18 MB, only the + smaller ones will be made. */ +void slabs_preallocate (unsigned int maxslabs); + /* Given object size, return id to use when allocating/freeing memory for object */ /* 0 means error: can't store such a large object */ unsigned int slabs_clsid(unsigned int size); diff -Nuar --exclude .svn --exclude CVS --exclude Makefile.in --exclude configure --exclude depcomp --exclude '*.orig' --exclude stamp-h.in --exclude 'config.*' --exclude aclocal.m4 --exclude Makefile memcached-1.1.12/slabs.c memcached-1.1.13-pre/slabs.c --- memcached-1.1.12/slabs.c 2004-04-26 14:26:48.000000000 -0700 +++ memcached-1.1.13-pre/slabs.c 2006-04-30 14:22:37.000000000 -0700 @@ -2,7 +2,7 @@ /* * Slabs memory allocation, based on powers-of-2 * - * $Id: slabs.c,v 1.15 2003/09/05 22:37:36 bradfitz Exp $ + * $Id: slabs.c 257 2006-03-23 06:41:44Z bradfitz $ */ #include @@ -83,6 +83,26 @@ slabclass[i].list_size = 0; slabclass[i].killing = 0; } + + slabs_preallocate(limit / POWER_BLOCK); +} + +void slabs_preallocate (unsigned int maxslabs) { + int i; + unsigned int prealloc = 0; + + /* pre-allocate a 1MB slab in every size class so people don't get + confused by non-intuitive "SERVER_ERROR out of memory" + messages. this is the most common question on the mailing + list. if you really don't want this, you can rebuild without + these three lines. */ + + for(i=POWER_SMALLEST; i<=POWER_LARGEST; i++) { + if (++prealloc > maxslabs) + return; + slabs_newslab(i); + } + } static int grow_slab_list (unsigned int id) {