libevent/buffer.c

   1 /*
   2  * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
   3  * Copyright (c) 2007-2010 Niels Provos and Nick Mathewson
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The name of the author may not be used to endorse or promote products
  14  *    derived from this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26  */
  27
  28 #include "event2/event-config.h"
  29
  30 #ifdef WIN32
  31 #include <winsock2.h>
  32 #include <windows.h>
  33 #include <io.h>
  34 #endif
  35
  36 #ifdef _EVENT_HAVE_VASPRINTF
  37 /* If we have vasprintf, we need to define this before we include stdio.h. */
  38 #define _GNU_SOURCE
  39 #endif
  40
  41 #include <sys/types.h>
  42
  43 #ifdef _EVENT_HAVE_SYS_TIME_H
  44 #include <sys/time.h>
  45 #endif
  46
  47 #ifdef _EVENT_HAVE_SYS_SOCKET_H
  48 #include <sys/socket.h>
  49 #endif
  50
  51 #ifdef _EVENT_HAVE_SYS_UIO_H
  52 #include <sys/uio.h>
  53 #endif
  54
  55 #ifdef _EVENT_HAVE_SYS_IOCTL_H
  56 #include <sys/ioctl.h>
  57 #endif
  58
  59 #ifdef _EVENT_HAVE_SYS_MMAN_H
  60 #include <sys/mman.h>
  61 #endif
  62
  63 #ifdef _EVENT_HAVE_SYS_SENDFILE_H
  64 #include <sys/sendfile.h>
  65 #endif
  66
  67 #include <errno.h>
  68 #include <stdio.h>
  69 #include <stdlib.h>
  70 #include <string.h>
  71 #ifdef _EVENT_HAVE_STDARG_H
  72 #include <stdarg.h>
  73 #endif
  74 #ifdef _EVENT_HAVE_UNISTD_H
  75 #include <unistd.h>
  76 #endif
  77 #include <limits.h>
  78
  79 #include "event2/event.h"
  80 #include "event2/buffer.h"
  81 #include "event2/buffer_compat.h"
  82 #include "event2/bufferevent.h"
  83 #include "event2/bufferevent_compat.h"
  84 #include "event2/bufferevent_struct.h"
  85 #include "event2/thread.h"
  86 #include "event2/event-config.h"
  87 #include "log-internal.h"
  88 #include "mm-internal.h"
  89 #include "util-internal.h"
  90 #include "evthread-internal.h"
  91 #include "evbuffer-internal.h"
  92 #include "bufferevent-internal.h"
  93
  94 /* some systems do not have MAP_FAILED */
  95 #ifndef MAP_FAILED
  96 #define MAP_FAILED      ((void *)-1)
  97 #endif
  98
  99 /* send file support */
 100 #if defined(_EVENT_HAVE_SYS_SENDFILE_H) && defined(_EVENT_HAVE_SENDFILE) && defined(__linux__)
 101 #define USE_SENDFILE            1
 102 #define SENDFILE_IS_LINUX       1
 103 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__FreeBSD__)
 104 #define USE_SENDFILE            1
 105 #define SENDFILE_IS_FREEBSD     1
 106 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__APPLE__)
 107 #define USE_SENDFILE            1
 108 #define SENDFILE_IS_MACOSX      1
 109 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
 110 #define USE_SENDFILE            1
 111 #define SENDFILE_IS_SOLARIS     1
 112 #endif
 113
 114 #ifdef USE_SENDFILE
 115 static int use_sendfile = 1;
 116 #endif
 117 #ifdef _EVENT_HAVE_MMAP
 118 static int use_mmap = 1;
 119 #endif
 120
 121
 122 /* Mask of user-selectable callback flags. */
 123 #define EVBUFFER_CB_USER_FLAGS      0xffff
 124 /* Mask of all internal-use-only flags. */
 125 #define EVBUFFER_CB_INTERNAL_FLAGS  0xffff0000
 126
 127 /* Flag set if the callback is using the cb_obsolete function pointer  */
 128 #define EVBUFFER_CB_OBSOLETE           0x00040000
 129
 130 /* evbuffer_chain support */
 131 #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
 132 #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
 133             0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))
 134
 135 #define CHAIN_PINNED(ch)  (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
 136 #define CHAIN_PINNED_R(ch)  (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)
 137
 138 static void evbuffer_chain_align(struct evbuffer_chain *chain);
 139 static int evbuffer_chain_should_realign(struct evbuffer_chain *chain,
 140     size_t datalen);
 141 static void evbuffer_deferred_callback(struct deferred_cb *cb, void *arg);
 142 static int evbuffer_ptr_memcmp(const struct evbuffer *buf,
 143     const struct evbuffer_ptr *pos, const char *mem, size_t len);
 144 static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf,
 145     size_t datlen);
 146
 147 #ifdef WIN32
 148 static int evbuffer_readfile(struct evbuffer *buf, evutil_socket_t fd,
 149     ev_ssize_t howmuch);
 150 #else
 151 #define evbuffer_readfile evbuffer_read
 152 #endif
 153
 154 static struct evbuffer_chain *
 155 evbuffer_chain_new(size_t size)
 156 {
 157         struct evbuffer_chain *chain;
 158         size_t to_alloc;
 159
 160         size += EVBUFFER_CHAIN_SIZE;
 161
 162         /* get the next largest memory that can hold the buffer */
 163         to_alloc = MIN_BUFFER_SIZE;
 164         while (to_alloc < size)
 165                 to_alloc <<= 1;
 166
 167         /* we get everything in one chunk */
 168         if ((chain = mm_malloc(to_alloc)) == NULL)
 169                 return (NULL);
 170
 171         memset(chain, 0, EVBUFFER_CHAIN_SIZE);
 172
 173         chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE;
 174
 175         /* this way we can manipulate the buffer to different addresses,
 176          * which is required for mmap for example.
 177          */
 178         chain->buffer = EVBUFFER_CHAIN_EXTRA(u_char, chain);
 179
 180         return (chain);
 181 }
 182
 183 static inline void
 184 evbuffer_chain_free(struct evbuffer_chain *chain)
 185 {
 186         if (CHAIN_PINNED(chain)) {
 187                 chain->flags |= EVBUFFER_DANGLING;
 188                 return;
 189         }
 190         if (chain->flags & (EVBUFFER_MMAP|EVBUFFER_SENDFILE|
 191                 EVBUFFER_REFERENCE)) {
 192                 if (chain->flags & EVBUFFER_REFERENCE) {
 193                         struct evbuffer_chain_reference *info =
 194                             EVBUFFER_CHAIN_EXTRA(
 195                                     struct evbuffer_chain_reference,
 196                                     chain);
 197                         if (info->cleanupfn)
 198                                 (*info->cleanupfn)(chain->buffer,
 199                                     chain->buffer_len,
 200                                     info->extra);
 201                 }
 202 #ifdef _EVENT_HAVE_MMAP
 203                 if (chain->flags & EVBUFFER_MMAP) {
 204                         struct evbuffer_chain_fd *info =
 205                             EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd,
 206                                 chain);
 207                         if (munmap(chain->buffer, chain->buffer_len) == -1)
 208                                 event_warn("%s: munmap failed", __func__);
 209                         if (close(info->fd) == -1)
 210                                 event_warn("%s: close(%d) failed",
 211                                     __func__, info->fd);
 212                 }
 213 #endif
 214 #ifdef USE_SENDFILE
 215                 if (chain->flags & EVBUFFER_SENDFILE) {
 216                         struct evbuffer_chain_fd *info =
 217                             EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd,
 218                                 chain);
 219                         if (close(info->fd) == -1)
 220                                 event_warn("%s: close(%d) failed",
 221                                     __func__, info->fd);
 222                 }
 223 #endif
 224         }
 225
 226         mm_free(chain);
 227 }
 228
 229 static void
 230 evbuffer_free_all_chains(struct evbuffer_chain *chain)
 231 {
 232         struct evbuffer_chain *next;
 233         for (; chain; chain = next) {
 234                 next = chain->next;
 235                 evbuffer_chain_free(chain);
 236         }
 237 }
 238
 239 #ifndef NDEBUG
 240 static int
 241 evbuffer_chains_all_empty(struct evbuffer_chain *chain)
 242 {
 243         for (; chain; chain = chain->next) {
 244                 if (chain->off)
 245                         return 0;
 246         }
 247         return 1;
 248 }
 249 #else
 250 /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
 251 "unused variable" warnings. */
 252 static inline int evbuffer_chains_all_empty(struct evbuffer_chain *chain) {
 253         return 1;
 254 }
 255 #endif
 256
 257 static void
 258 evbuffer_chain_insert(struct evbuffer *buf,
 259     struct evbuffer_chain *chain)
 260 {
 261         ASSERT_EVBUFFER_LOCKED(buf);
 262         if (*buf->last_with_datap == NULL) {
 263                 /* There are no chains data on the buffer at all. */
 264                 EVUTIL_ASSERT(buf->last_with_datap == &buf->first);
 265                 EVUTIL_ASSERT(buf->first == NULL);
 266                 buf->first = buf->last = chain;
 267         } else {
 268                 struct evbuffer_chain **ch = buf->last_with_datap;
 269                 /* Find the first victim chain.  It might be *last_with_datap */
 270                 while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch)))
 271                         ch = &(*ch)->next;
 272                 if (*ch == NULL) {
 273                         /* There is no victim; just append this new chain. */
 274                         buf->last->next = chain;
 275                         if (chain->off)
 276                                 buf->last_with_datap = &buf->last->next;
 277                 } else {
 278                         /* Replace all victim chains with this chain. */
 279                         EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch));
 280                         evbuffer_free_all_chains(*ch);
 281                         *ch = chain;
 282                 }
 283                 buf->last = chain;
 284         }
 285         buf->total_len += chain->off;
 286 }
 287
 288 static inline struct evbuffer_chain *
 289 evbuffer_chain_insert_new(struct evbuffer *buf, size_t datlen)
 290 {
 291         struct evbuffer_chain *chain;
 292         if ((chain = evbuffer_chain_new(datlen)) == NULL)
 293                 return NULL;
 294         evbuffer_chain_insert(buf, chain);
 295         return chain;
 296 }
 297
 298 void
 299 _evbuffer_chain_pin(struct evbuffer_chain *chain, unsigned flag)
 300 {
 301         EVUTIL_ASSERT((chain->flags & flag) == 0);
 302         chain->flags |= flag;
 303 }
 304
 305 void
 306 _evbuffer_chain_unpin(struct evbuffer_chain *chain, unsigned flag)
 307 {
 308         EVUTIL_ASSERT((chain->flags & flag) != 0);
 309         chain->flags &= ~flag;
 310         if (chain->flags & EVBUFFER_DANGLING)
 311                 evbuffer_chain_free(chain);
 312 }
 313
 314 struct evbuffer *
 315 evbuffer_new(void)
 316 {
 317         struct evbuffer *buffer;
 318
 319         buffer = mm_calloc(1, sizeof(struct evbuffer));
 320         if (buffer == NULL)
 321                 return (NULL);
 322
 323         TAILQ_INIT(&buffer->callbacks);
 324         buffer->refcnt = 1;
 325         buffer->last_with_datap = &buffer->first;
 326
 327         return (buffer);
 328 }
 329
 330 void
 331 _evbuffer_incref(struct evbuffer *buf)
 332 {
 333         EVBUFFER_LOCK(buf);
 334         ++buf->refcnt;
 335         EVBUFFER_UNLOCK(buf);
 336 }
 337
 338 void
 339 _evbuffer_incref_and_lock(struct evbuffer *buf)
 340 {
 341         EVBUFFER_LOCK(buf);
 342         ++buf->refcnt;
 343 }
 344
 345 int
 346 evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base)
 347 {
 348         EVBUFFER_LOCK(buffer);
 349         buffer->cb_queue = event_base_get_deferred_cb_queue(base);
 350         buffer->deferred_cbs = 1;
 351         event_deferred_cb_init(&buffer->deferred,
 352             evbuffer_deferred_callback, buffer);
 353         EVBUFFER_UNLOCK(buffer);
 354         return 0;
 355 }
 356
 357 int
 358 evbuffer_enable_locking(struct evbuffer *buf, void *lock)
 359 {
 360 #ifdef _EVENT_DISABLE_THREAD_SUPPORT
 361         return -1;
 362 #else
 363         if (buf->lock)
 364                 return -1;
 365
 366         if (!lock) {
 367                 EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE);
 368                 if (!lock)
 369                         return -1;
 370                 buf->lock = lock;
 371                 buf->own_lock = 1;
 372         } else {
 373                 buf->lock = lock;
 374                 buf->own_lock = 0;
 375         }
 376
 377         return 0;
 378 #endif
 379 }
 380
 381 void
 382 evbuffer_set_parent(struct evbuffer *buf, struct bufferevent *bev)
 383 {
 384         EVBUFFER_LOCK(buf);
 385         buf->parent = bev;
 386         EVBUFFER_UNLOCK(buf);
 387 }
 388
 389 static void
 390 evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred)
 391 {
 392         struct evbuffer_cb_entry *cbent, *next;
 393         struct evbuffer_cb_info info;
 394         size_t new_size;
 395         ev_uint32_t mask, masked_val;
 396         int clear = 1;
 397
 398         if (running_deferred) {
 399                 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
 400                 masked_val = EVBUFFER_CB_ENABLED;
 401         } else if (buffer->deferred_cbs) {
 402                 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
 403                 masked_val = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
 404                 /* Don't zero-out n_add/n_del, since the deferred callbacks
 405                    will want to see them. */
 406                 clear = 0;
 407         } else {
 408                 mask = EVBUFFER_CB_ENABLED;
 409                 masked_val = EVBUFFER_CB_ENABLED;
 410         }
 411
 412         ASSERT_EVBUFFER_LOCKED(buffer);
 413
 414         if (TAILQ_EMPTY(&buffer->callbacks)) {
 415                 buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
 416                 return;
 417         }
 418         if (buffer->n_add_for_cb == 0 && buffer->n_del_for_cb == 0)
 419                 return;
 420
 421         new_size = buffer->total_len;
 422         info.orig_size = new_size + buffer->n_del_for_cb - buffer->n_add_for_cb;
 423         info.n_added = buffer->n_add_for_cb;
 424         info.n_deleted = buffer->n_del_for_cb;
 425         if (clear) {
 426                 buffer->n_add_for_cb = 0;
 427                 buffer->n_del_for_cb = 0;
 428         }
 429         for (cbent = TAILQ_FIRST(&buffer->callbacks);
 430              cbent != TAILQ_END(&buffer->callbacks);
 431              cbent = next) {
 432                 /* Get the 'next' pointer now in case this callback decides
 433                  * to remove itself or something. */
 434                 next = TAILQ_NEXT(cbent, next);
 435
 436                 if ((cbent->flags & mask) != masked_val)
 437                         continue;
 438
 439                 if ((cbent->flags & EVBUFFER_CB_OBSOLETE))
 440                         cbent->cb.cb_obsolete(buffer,
 441                             info.orig_size, new_size, cbent->cbarg);
 442                 else
 443                         cbent->cb.cb_func(buffer, &info, cbent->cbarg);
 444         }
 445 }
 446
 447 static inline void
 448 evbuffer_invoke_callbacks(struct evbuffer *buffer)
 449 {
 450         if (buffer->deferred_cbs) {
 451                 if (buffer->deferred.queued)
 452                         return;
 453                 _evbuffer_incref_and_lock(buffer);
 454                 if (buffer->parent)
 455                         bufferevent_incref(buffer->parent);
 456                 EVBUFFER_UNLOCK(buffer);
 457                 event_deferred_cb_schedule(buffer->cb_queue, &buffer->deferred);
 458         }
 459
 460         evbuffer_run_callbacks(buffer, 0);
 461 }
 462
 463 static void
 464 evbuffer_deferred_callback(struct deferred_cb *cb, void *arg)
 465 {
 466         struct bufferevent *parent = NULL;
 467         struct evbuffer *buffer = arg;
 468
 469         /* XXXX It would be better to run these callbacks without holding the
 470          * lock */
 471         EVBUFFER_LOCK(buffer);
 472         parent = buffer->parent;
 473         evbuffer_run_callbacks(buffer, 1);
 474         _evbuffer_decref_and_unlock(buffer);
 475         if (parent)
 476                 bufferevent_decref(parent);
 477 }
 478
 479 static void
 480 evbuffer_remove_all_callbacks(struct evbuffer *buffer)
 481 {
 482         struct evbuffer_cb_entry *cbent;
 483
 484         while ((cbent = TAILQ_FIRST(&buffer->callbacks))) {
 485             TAILQ_REMOVE(&buffer->callbacks, cbent, next);
 486             mm_free(cbent);
 487         }
 488 }
 489
 490 void
 491 _evbuffer_decref_and_unlock(struct evbuffer *buffer)
 492 {
 493         struct evbuffer_chain *chain, *next;
 494         ASSERT_EVBUFFER_LOCKED(buffer);
 495
 496         EVUTIL_ASSERT(buffer->refcnt > 0);
 497
 498         if (--buffer->refcnt > 0) {
 499                 EVBUFFER_UNLOCK(buffer);
 500                 return;
 501         }
 502
 503         for (chain = buffer->first; chain != NULL; chain = next) {
 504                 next = chain->next;
 505                 evbuffer_chain_free(chain);
 506         }
 507         evbuffer_remove_all_callbacks(buffer);
 508         if (buffer->deferred_cbs)
 509                 event_deferred_cb_cancel(buffer->cb_queue, &buffer->deferred);
 510
 511         EVBUFFER_UNLOCK(buffer);
 512         if (buffer->own_lock)
 513                 EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE);
 514         mm_free(buffer);
 515 }
 516
 517 void
 518 evbuffer_free(struct evbuffer *buffer)
 519 {
 520         EVBUFFER_LOCK(buffer);
 521         _evbuffer_decref_and_unlock(buffer);
 522 }
 523
 524 void
 525 evbuffer_lock(struct evbuffer *buf)
 526 {
 527         EVBUFFER_LOCK(buf);
 528 }
 529
 530 void
 531 evbuffer_unlock(struct evbuffer *buf)
 532 {
 533         EVBUFFER_UNLOCK(buf);
 534 }
 535
 536 size_t
 537 evbuffer_get_length(const struct evbuffer *buffer)
 538 {
 539         size_t result;
 540
 541         EVBUFFER_LOCK(buffer);
 542
 543         result = (buffer->total_len);
 544
 545         EVBUFFER_UNLOCK(buffer);
 546
 547         return result;
 548 }
 549
 550 size_t
 551 evbuffer_get_contiguous_space(const struct evbuffer *buf)
 552 {
 553         struct evbuffer_chain *chain;
 554         size_t result;
 555
 556         EVBUFFER_LOCK(buf);
 557         chain = buf->first;
 558         result = (chain != NULL ? chain->off : 0);
 559         EVBUFFER_UNLOCK(buf);
 560
 561         return result;
 562 }
 563
 564 int
 565 evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size,
 566     struct evbuffer_iovec *vec, int n_vecs)
 567 {
 568         struct evbuffer_chain *chain, **chainp;
 569         int n = -1;
 570
 571         EVBUFFER_LOCK(buf);
 572         if (buf->freeze_end)
 573                 goto done;
 574         if (n_vecs < 1)
 575                 goto done;
 576         if (n_vecs == 1) {
 577                 if ((chain = evbuffer_expand_singlechain(buf, size)) == NULL)
 578                         goto done;
 579
 580                 vec[0].iov_base = CHAIN_SPACE_PTR(chain);
 581                 vec[0].iov_len = (size_t) CHAIN_SPACE_LEN(chain);
 582                 EVUTIL_ASSERT(size<0 || (size_t)vec[0].iov_len >= (size_t)size);
 583                 n = 1;
 584         } else {
 585                 if (_evbuffer_expand_fast(buf, size, n_vecs)<0)
 586                         goto done;
 587                 n = _evbuffer_read_setup_vecs(buf, size, vec, n_vecs,
 588                                 &chainp, 0);
 589         }
 590
 591 done:
 592         EVBUFFER_UNLOCK(buf);
 593         return n;
 594
 595 }
 596
 597 static int
 598 advance_last_with_data(struct evbuffer *buf)
 599 {
 600         int n = 0;
 601         ASSERT_EVBUFFER_LOCKED(buf);
 602
 603         if (!*buf->last_with_datap)
 604                 return 0;
 605
 606         while ((*buf->last_with_datap)->next && (*buf->last_with_datap)->next->off) {
 607                 buf->last_with_datap = &(*buf->last_with_datap)->next;
 608                 ++n;
 609         }
 610         return n;
 611 }
 612
 613 int
 614 evbuffer_commit_space(struct evbuffer *buf,
 615     struct evbuffer_iovec *vec, int n_vecs)
 616 {
 617         struct evbuffer_chain *chain, **firstchainp, **chainp;
 618         int result = -1;
 619         size_t added = 0;
 620         int i;
 621
 622         EVBUFFER_LOCK(buf);
 623
 624         if (buf->freeze_end)
 625                 goto done;
 626         if (n_vecs == 0) {
 627                 result = 0;
 628                 goto done;
 629         } else if (n_vecs == 1 &&
 630             (buf->last && vec[0].iov_base == (void*)CHAIN_SPACE_PTR(buf->last))) {
 631                 /* The user only got or used one chain; it might not
 632                  * be the first one with space in it. */
 633                 if ((size_t)vec[0].iov_len > (size_t)CHAIN_SPACE_LEN(buf->last))
 634                         goto done;
 635                 buf->last->off += vec[0].iov_len;
 636                 added = vec[0].iov_len;
 637                 if (added)
 638                         advance_last_with_data(buf);
 639                 goto okay;
 640         }
 641
 642         /* Advance 'firstchain' to the first chain with space in it. */
 643         firstchainp = buf->last_with_datap;
 644         if (!*firstchainp)
 645                 goto done;
 646         if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
 647                 firstchainp = &(*firstchainp)->next;
 648         }
 649
 650         chain = *firstchainp;
 651         /* pass 1: make sure that the pointers and lengths of vecs[] are in
 652          * bounds before we try to commit anything. */
 653         for (i=0; i<n_vecs; ++i) {
 654                 if (!chain)
 655                         goto done;
 656                 if (vec[i].iov_base != (void*)CHAIN_SPACE_PTR(chain) ||
 657                     (size_t)vec[i].iov_len > CHAIN_SPACE_LEN(chain))
 658                         goto done;
 659                 chain = chain->next;
 660         }
 661         /* pass 2: actually adjust all the chains. */
 662         chainp = firstchainp;
 663         for (i=0; i<n_vecs; ++i) {
 664                 (*chainp)->off += vec[i].iov_len;
 665                 added += vec[i].iov_len;
 666                 if (vec[i].iov_len) {
 667                         buf->last_with_datap = chainp;
 668                 }
 669                 chainp = &(*chainp)->next;
 670         }
 671
 672 okay:
 673         buf->total_len += added;
 674         buf->n_add_for_cb += added;
 675         result = 0;
 676         evbuffer_invoke_callbacks(buf);
 677
 678 done:
 679         EVBUFFER_UNLOCK(buf);
 680         return result;
 681 }
 682
 683 static inline int
 684 HAS_PINNED_R(struct evbuffer *buf)
 685 {
 686         return (buf->last && CHAIN_PINNED_R(buf->last));
 687 }
 688
 689 static inline void
 690 ZERO_CHAIN(struct evbuffer *dst)
 691 {
 692         ASSERT_EVBUFFER_LOCKED(dst);
 693         dst->first = NULL;
 694         dst->last = NULL;
 695         dst->last_with_datap = &(dst)->first;
 696         dst->total_len = 0;
 697 }
 698
 699 /* Prepares the contents of src to be moved to another buffer by removing
 700  * read-pinned chains. The first pinned chain is saved in first, and the
 701  * last in last. If src has no read-pinned chains, first and last are set
 702  * to NULL. */
 703 static int
 704 PRESERVE_PINNED(struct evbuffer *src, struct evbuffer_chain **first,
 705                 struct evbuffer_chain **last)
 706 {
 707         struct evbuffer_chain *chain, **pinned;
 708
 709         ASSERT_EVBUFFER_LOCKED(src);
 710
 711         if (!HAS_PINNED_R(src)) {
 712                 *first = *last = NULL;
 713                 return 0;
 714         }
 715
 716         pinned = src->last_with_datap;
 717         if (!CHAIN_PINNED_R(*pinned))
 718                 pinned = &(*pinned)->next;
 719         EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned));
 720         chain = *first = *pinned;
 721         *last = src->last;
 722
 723         /* If there's data in the first pinned chain, we need to allocate
 724          * a new chain and copy the data over. */
 725         if (chain->off) {
 726                 struct evbuffer_chain *tmp;
 727
 728                 EVUTIL_ASSERT(pinned == src->last_with_datap);
 729                 tmp = evbuffer_chain_new(chain->off);
 730                 if (!tmp)
 731                         return -1;
 732                 memcpy(tmp->buffer, chain->buffer + chain->misalign,
 733                         chain->off);
 734                 tmp->off = chain->off;
 735                 *src->last_with_datap = tmp;
 736                 src->last = tmp;
 737                 chain->misalign += chain->off;
 738                 chain->off = 0;
 739         } else {
 740                 src->last = *src->last_with_datap;
 741                 *pinned = NULL;
 742         }
 743
 744         return 0;
 745 }
 746
 747 static inline void
 748 RESTORE_PINNED(struct evbuffer *src, struct evbuffer_chain *pinned,
 749                 struct evbuffer_chain *last)
 750 {
 751         ASSERT_EVBUFFER_LOCKED(src);
 752
 753         if (!pinned) {
 754                 ZERO_CHAIN(src);
 755                 return;
 756         }
 757
 758         src->first = pinned;
 759         src->last = last;
 760         src->last_with_datap = &src->first;
 761         src->total_len = 0;
 762 }
 763
 764 static inline void
 765 COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src)
 766 {
 767         ASSERT_EVBUFFER_LOCKED(dst);
 768         ASSERT_EVBUFFER_LOCKED(src);
 769         dst->first = src->first;
 770         if (src->last_with_datap == &src->first)
 771                 dst->last_with_datap = &dst->first;
 772         else
 773                 dst->last_with_datap = src->last_with_datap;
 774         dst->last = src->last;
 775         dst->total_len = src->total_len;
 776 }
 777
 778 static void
 779 APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
 780 {
 781         ASSERT_EVBUFFER_LOCKED(dst);
 782         ASSERT_EVBUFFER_LOCKED(src);
 783         dst->last->next = src->first;
 784         if (src->last_with_datap == &src->first)
 785                 dst->last_with_datap = &dst->last->next;
 786         else
 787                 dst->last_with_datap = src->last_with_datap;
 788         dst->last = src->last;
 789         dst->total_len += src->total_len;
 790 }
 791
 792 static void
 793 PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
 794 {
 795         ASSERT_EVBUFFER_LOCKED(dst);
 796         ASSERT_EVBUFFER_LOCKED(src);
 797         src->last->next = dst->first;
 798         dst->first = src->first;
 799         dst->total_len += src->total_len;
 800         if (*dst->last_with_datap == NULL) {
 801                 if (src->last_with_datap == &(src)->first)
 802                         dst->last_with_datap = &dst->first;
 803                 else
 804                         dst->last_with_datap = src->last_with_datap;
 805         } else if (dst->last_with_datap == &dst->first) {
 806                 dst->last_with_datap = &src->last->next;
 807         }
 808 }
 809
 810 int
 811 evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
 812 {
 813         struct evbuffer_chain *pinned, *last;
 814         size_t in_total_len, out_total_len;
 815         int result = 0;
 816
 817         EVBUFFER_LOCK2(inbuf, outbuf);
 818         in_total_len = inbuf->total_len;
 819         out_total_len = outbuf->total_len;
 820
 821         if (in_total_len == 0 || outbuf == inbuf)
 822                 goto done;
 823
 824         if (outbuf->freeze_end || inbuf->freeze_start) {
 825                 result = -1;
 826                 goto done;
 827         }
 828
 829         if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
 830                 result = -1;
 831                 goto done;
 832         }
 833
 834         if (out_total_len == 0) {
 835                 /* There might be an empty chain at the start of outbuf; free
 836                  * it. */
 837                 evbuffer_free_all_chains(outbuf->first);
 838                 COPY_CHAIN(outbuf, inbuf);
 839         } else {
 840                 APPEND_CHAIN(outbuf, inbuf);
 841         }
 842
 843         RESTORE_PINNED(inbuf, pinned, last);
 844
 845         inbuf->n_del_for_cb += in_total_len;
 846         outbuf->n_add_for_cb += in_total_len;
 847
 848         evbuffer_invoke_callbacks(inbuf);
 849         evbuffer_invoke_callbacks(outbuf);
 850
 851 done:
 852         EVBUFFER_UNLOCK2(inbuf, outbuf);
 853         return result;
 854 }
 855
 856 int
 857 evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
 858 {
 859         struct evbuffer_chain *pinned, *last;
 860         size_t in_total_len, out_total_len;
 861         int result = 0;
 862
 863         EVBUFFER_LOCK2(inbuf, outbuf);
 864
 865         in_total_len = inbuf->total_len;
 866         out_total_len = outbuf->total_len;
 867
 868         if (!in_total_len || inbuf == outbuf)
 869                 goto done;
 870
 871         if (outbuf->freeze_start || inbuf->freeze_start) {
 872                 result = -1;
 873                 goto done;
 874         }
 875
 876         if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
 877                 result = -1;
 878                 goto done;
 879         }
 880
 881         if (out_total_len == 0) {
 882                 /* There might be an empty chain at the start of outbuf; free
 883                  * it. */
 884                 evbuffer_free_all_chains(outbuf->first);
 885                 COPY_CHAIN(outbuf, inbuf);
 886         } else {
 887                 PREPEND_CHAIN(outbuf, inbuf);
 888         }
 889
 890         RESTORE_PINNED(inbuf, pinned, last);
 891
 892         inbuf->n_del_for_cb += in_total_len;
 893         outbuf->n_add_for_cb += in_total_len;
 894
 895         evbuffer_invoke_callbacks(inbuf);
 896         evbuffer_invoke_callbacks(outbuf);
 897 done:
 898         EVBUFFER_UNLOCK2(inbuf, outbuf);
 899         return result;
 900 }
 901
 902 int
 903 evbuffer_drain(struct evbuffer *buf, size_t len)
 904 {
 905         struct evbuffer_chain *chain, *next;
 906         size_t remaining, old_len;
 907         int result = 0;
 908
 909         EVBUFFER_LOCK(buf);
 910         old_len = buf->total_len;
 911
 912         if (old_len == 0)
 913                 goto done;
 914
 915         if (buf->freeze_start) {
 916                 result = -1;
 917                 goto done;
 918         }
 919
 920         if (len >= old_len && !HAS_PINNED_R(buf)) {
 921                 len = old_len;
 922                 for (chain = buf->first; chain != NULL; chain = next) {
 923                         next = chain->next;
 924                         evbuffer_chain_free(chain);
 925                 }
 926
 927                 ZERO_CHAIN(buf);
 928         } else {
 929                 if (len >= old_len)
 930                         len = old_len;
 931
 932                 buf->total_len -= len;
 933                 remaining = len;
 934                 for (chain = buf->first;
 935                      remaining >= chain->off;
 936                      chain = next) {
 937                         next = chain->next;
 938                         remaining -= chain->off;
 939
 940                         if (chain == *buf->last_with_datap) {
 941                                 buf->last_with_datap = &buf->first;
 942                         }
 943                         if (&chain->next == buf->last_with_datap)
 944                                 buf->last_with_datap = &buf->first;
 945
 946                         if (CHAIN_PINNED_R(chain)) {
 947                                 EVUTIL_ASSERT(remaining == 0);
 948                                 chain->misalign += chain->off;
 949                                 chain->off = 0;
 950                                 break;
 951                         } else
 952                                 evbuffer_chain_free(chain);
 953                 }
 954
 955                 buf->first = chain;
 956                 if (chain) {
 957                         chain->misalign += remaining;
 958                         chain->off -= remaining;
 959                 }
 960         }
 961
 962         buf->n_del_for_cb += len;
 963         /* Tell someone about changes in this buffer */
 964         evbuffer_invoke_callbacks(buf);
 965
 966 done:
 967         EVBUFFER_UNLOCK(buf);
 968         return result;
 969 }
 970
 971 /* Reads data from an event buffer and drains the bytes read */
 972 int
 973 evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen)
 974 {
 975         ev_ssize_t n;
 976         EVBUFFER_LOCK(buf);
 977         n = evbuffer_copyout(buf, data_out, datlen);
 978         if (n > 0) {
 979                 if (evbuffer_drain(buf, n)<0)
 980                         n = -1;
 981         }
 982         EVBUFFER_UNLOCK(buf);
 983         return (int)n;
 984 }
 985
 986 ev_ssize_t
 987 evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen)
 988 {
 989         /*XXX fails badly on sendfile case. */
 990         struct evbuffer_chain *chain;
 991         char *data = data_out;
 992         size_t nread;
 993         ev_ssize_t result = 0;
 994
 995         EVBUFFER_LOCK(buf);
 996
 997         chain = buf->first;
 998
 999         if (datlen >= buf->total_len)
1000                 datlen = buf->total_len;
1001
1002         if (datlen == 0)
1003                 goto done;
1004
1005         if (buf->freeze_start) {
1006                 result = -1;
1007                 goto done;
1008         }
1009
1010         nread = datlen;
1011
1012         while (datlen && datlen >= chain->off) {
1013                 memcpy(data, chain->buffer + chain->misalign, chain->off);
1014                 data += chain->off;
1015                 datlen -= chain->off;
1016
1017                 chain = chain->next;
1018                 EVUTIL_ASSERT(chain || datlen==0);
1019         }
1020
1021         if (datlen) {
1022                 EVUTIL_ASSERT(chain);
1023                 memcpy(data, chain->buffer + chain->misalign, datlen);
1024         }
1025
1026         result = nread;
1027 done:
1028         EVBUFFER_UNLOCK(buf);
1029         return result;
1030 }
1031
1032 /* reads data from the src buffer to the dst buffer, avoids memcpy as
1033  * possible. */
1034 /*  XXXX should return ev_ssize_t */
1035 int
1036 evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst,
1037     size_t datlen)
1038 {
1039         /*XXX We should have an option to force this to be zero-copy.*/
1040
1041         /*XXX can fail badly on sendfile case. */
1042         struct evbuffer_chain *chain, *previous;
1043         size_t nread = 0;
1044         int result;
1045
1046         EVBUFFER_LOCK2(src, dst);
1047
1048         chain = previous = src->first;
1049
1050         if (datlen == 0 || dst == src) {
1051                 result = 0;
1052                 goto done;
1053         }
1054
1055         if (dst->freeze_end || src->freeze_start) {
1056                 result = -1;
1057                 goto done;
1058         }
1059
1060         /* short-cut if there is no more data buffered */
1061         if (datlen >= src->total_len) {
1062                 datlen = src->total_len;
1063                 evbuffer_add_buffer(dst, src);
1064                 result = (int)datlen; /*XXXX should return ev_ssize_t*/
1065                 goto done;
1066         }
1067
1068         /* removes chains if possible */
1069         while (chain->off <= datlen) {
1070                 /* We can't remove the last with data from src unless we
1071                  * remove all chains, in which case we would have done the if
1072                  * block above */
1073                 EVUTIL_ASSERT(chain != *src->last_with_datap);
1074                 nread += chain->off;
1075                 datlen -= chain->off;
1076                 previous = chain;
1077                 if (src->last_with_datap == &chain->next)
1078                         src->last_with_datap = &src->first;
1079                 chain = chain->next;
1080         }
1081
1082         if (nread) {
1083                 /* we can remove the chain */
1084                 if (dst->first == NULL) {
1085                         dst->first = src->first;
1086                 } else {
1087                         dst->last->next = src->first;
1088                 }
1089                 dst->last = previous;
1090                 previous->next = NULL;
1091                 src->first = chain;
1092                 advance_last_with_data(dst);
1093
1094                 dst->total_len += nread;
1095                 dst->n_add_for_cb += nread;
1096         }
1097
1098         /* we know that there is more data in the src buffer than
1099          * we want to read, so we manually drain the chain */
1100         evbuffer_add(dst, chain->buffer + chain->misalign, datlen);
1101         chain->misalign += datlen;
1102         chain->off -= datlen;
1103         nread += datlen;
1104
1105         src->total_len -= nread;
1106         src->n_del_for_cb += nread;
1107
1108         if (nread) {
1109                 evbuffer_invoke_callbacks(dst);
1110                 evbuffer_invoke_callbacks(src);
1111         }
1112         result = (int)nread;/*XXXX should change return type */
1113
1114 done:
1115         EVBUFFER_UNLOCK2(src, dst);
1116         return result;
1117 }
1118
1119 unsigned char *
1120 evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size)
1121 {
1122         struct evbuffer_chain *chain, *next, *tmp, *last_with_data;
1123         unsigned char *buffer, *result = NULL;
1124         ev_ssize_t remaining;
1125         int removed_last_with_data = 0;
1126         int removed_last_with_datap = 0;
1127
1128         EVBUFFER_LOCK(buf);
1129
1130         chain = buf->first;
1131
1132         if (size < 0)
1133                 size = buf->total_len;
1134         /* if size > buf->total_len, we cannot guarantee to the user that she
1135          * is going to have a long enough buffer afterwards; so we return
1136          * NULL */
1137         if (size == 0 || (size_t)size > buf->total_len)
1138                 goto done;
1139
1140         /* No need to pull up anything; the first size bytes are
1141          * already here. */
1142         if (chain->off >= (size_t)size) {
1143                 result = chain->buffer + chain->misalign;
1144                 goto done;
1145         }
1146
1147         /* Make sure that none of the chains we need to copy from is pinned. */
1148         remaining = size - chain->off;
1149         EVUTIL_ASSERT(remaining >= 0);
1150         for (tmp=chain->next; tmp; tmp=tmp->next) {
1151                 if (CHAIN_PINNED(tmp))
1152                         goto done;
1153                 if (tmp->off >= (size_t)remaining)
1154                         break;
1155                 remaining -= tmp->off;
1156         }
1157
1158         if (CHAIN_PINNED(chain)) {
1159                 size_t old_off = chain->off;
1160                 if (CHAIN_SPACE_LEN(chain) < size - chain->off) {
1161                         /* not enough room at end of chunk. */
1162                         goto done;
1163                 }
1164                 buffer = CHAIN_SPACE_PTR(chain);
1165                 tmp = chain;
1166                 tmp->off = size;
1167                 size -= old_off;
1168                 chain = chain->next;
1169         } else if (chain->buffer_len - chain->misalign >= (size_t)size) {
1170                 /* already have enough space in the first chain */
1171                 size_t old_off = chain->off;
1172                 buffer = chain->buffer + chain->misalign + chain->off;
1173                 tmp = chain;
1174                 tmp->off = size;
1175                 size -= old_off;
1176                 chain = chain->next;
1177         } else {
1178                 if ((tmp = evbuffer_chain_new(size)) == NULL) {
1179                         event_warn("%s: out of memory", __func__);
1180                         goto done;
1181                 }
1182                 buffer = tmp->buffer;
1183                 tmp->off = size;
1184                 buf->first = tmp;
1185         }
1186
1187         /* TODO(niels): deal with buffers that point to NULL like sendfile */
1188
1189         /* Copy and free every chunk that will be entirely pulled into tmp */
1190         last_with_data = *buf->last_with_datap;
1191         for (; chain != NULL && (size_t)size >= chain->off; chain = next) {
1192                 next = chain->next;
1193
1194                 memcpy(buffer, chain->buffer + chain->misalign, chain->off);
1195                 size -= chain->off;
1196                 buffer += chain->off;
1197                 if (chain == last_with_data)
1198                         removed_last_with_data = 1;
1199                 if (&chain->next == buf->last_with_datap)
1200                         removed_last_with_datap = 1;
1201
1202                 evbuffer_chain_free(chain);
1203         }
1204
1205         if (chain != NULL) {
1206                 memcpy(buffer, chain->buffer + chain->misalign, size);
1207                 chain->misalign += size;
1208                 chain->off -= size;
1209         } else {
1210                 buf->last = tmp;
1211         }
1212
1213         tmp->next = chain;
1214
1215         if (removed_last_with_data) {
1216                 buf->last_with_datap = &buf->first;
1217         } else if (removed_last_with_datap) {
1218                 if (buf->first->next && buf->first->next->off)
1219                         buf->last_with_datap = &buf->first->next;
1220                 else
1221                         buf->last_with_datap = &buf->first;
1222         }
1223
1224         result = (tmp->buffer + tmp->misalign);
1225
1226 done:
1227         EVBUFFER_UNLOCK(buf);
1228         return result;
1229 }
1230
1231 /*
1232  * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
1233  * The returned buffer needs to be freed by the called.
1234  */
1235 char *
1236 evbuffer_readline(struct evbuffer *buffer)
1237 {
1238         return evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY);
1239 }
1240
1241 static inline ev_ssize_t
1242 evbuffer_strchr(struct evbuffer_ptr *it, const char chr)
1243 {
1244         struct evbuffer_chain *chain = it->_internal.chain;
1245         size_t i = it->_internal.pos_in_chain;
1246         while (chain != NULL) {
1247                 char *buffer = (char *)chain->buffer + chain->misalign;
1248                 char *cp = memchr(buffer+i, chr, chain->off-i);
1249                 if (cp) {
1250                         it->_internal.chain = chain;
1251                         it->_internal.pos_in_chain = cp - buffer;
1252                         it->pos += (cp - buffer);
1253                         return it->pos;
1254                 }
1255                 it->pos += chain->off - i;
1256                 i = 0;
1257                 chain = chain->next;
1258         }
1259
1260         return (-1);
1261 }
1262
1263 static inline char *
1264 find_eol_char(char *s, size_t len)
1265 {
1266 #define CHUNK_SZ 128
1267         /* Lots of benchmarking found this approach to be faster in practice
1268          * than doing two memchrs over the whole buffer, doin a memchr on each
1269          * char of the buffer, or trying to emulate memchr by hand. */
1270         char *s_end, *cr, *lf;
1271         s_end = s+len;
1272         while (s < s_end) {
1273                 size_t chunk = (s + CHUNK_SZ < s_end) ? CHUNK_SZ : (s_end - s);
1274                 cr = memchr(s, '\r', chunk);
1275                 lf = memchr(s, '\n', chunk);
1276                 if (cr) {
1277                         if (lf && lf < cr)
1278                                 return lf;
1279                         return cr;
1280                 } else if (lf) {
1281                         return lf;
1282                 }
1283                 s += CHUNK_SZ;
1284         }
1285
1286         return NULL;
1287 #undef CHUNK_SZ
1288 }
1289
1290 static ev_ssize_t
1291 evbuffer_find_eol_char(struct evbuffer_ptr *it)
1292 {
1293         struct evbuffer_chain *chain = it->_internal.chain;
1294         size_t i = it->_internal.pos_in_chain;
1295         while (chain != NULL) {
1296                 char *buffer = (char *)chain->buffer + chain->misalign;
1297                 char *cp = find_eol_char(buffer+i, chain->off-i);
1298                 if (cp) {
1299                         it->_internal.chain = chain;
1300                         it->_internal.pos_in_chain = cp - buffer;
1301                         it->pos += (cp - buffer) - i;
1302                         return it->pos;
1303                 }
1304                 it->pos += chain->off - i;
1305                 i = 0;
1306                 chain = chain->next;
1307         }
1308
1309         return (-1);
1310 }
1311
1312 static inline int
1313 evbuffer_strspn(
1314         struct evbuffer_ptr *ptr, const char *chrset)
1315 {
1316         int count = 0;
1317         struct evbuffer_chain *chain = ptr->_internal.chain;
1318         size_t i = ptr->_internal.pos_in_chain;
1319
1320         if (!chain)
1321                 return -1;
1322
1323         while (1) {
1324                 char *buffer = (char *)chain->buffer + chain->misalign;
1325                 for (; i < chain->off; ++i) {
1326                         const char *p = chrset;
1327                         while (*p) {
1328                                 if (buffer[i] == *p++)
1329                                         goto next;
1330                         }
1331                         ptr->_internal.chain = chain;
1332                         ptr->_internal.pos_in_chain = i;
1333                         ptr->pos += count;
1334                         return count;
1335                 next:
1336                         ++count;
1337                 }
1338                 i = 0;
1339
1340                 if (! chain->next) {
1341                         ptr->_internal.chain = chain;
1342                         ptr->_internal.pos_in_chain = i;
1343                         ptr->pos += count;
1344                         return count;
1345                 }
1346
1347                 chain = chain->next;
1348         }
1349 }
1350
1351
1352 static inline char
1353 evbuffer_getchr(struct evbuffer_ptr *it)
1354 {
1355         struct evbuffer_chain *chain = it->_internal.chain;
1356         size_t off = it->_internal.pos_in_chain;
1357
1358         return chain->buffer[chain->misalign + off];
1359 }
1360
1361 struct evbuffer_ptr
1362 evbuffer_search_eol(struct evbuffer *buffer,
1363     struct evbuffer_ptr *start, size_t *eol_len_out,
1364     enum evbuffer_eol_style eol_style)
1365 {
1366         struct evbuffer_ptr it, it2;
1367         size_t extra_drain = 0;
1368         int ok = 0;
1369
1370         EVBUFFER_LOCK(buffer);
1371
1372         if (start) {
1373                 memcpy(&it, start, sizeof(it));
1374         } else {
1375                 it.pos = 0;
1376                 it._internal.chain = buffer->first;
1377                 it._internal.pos_in_chain = 0;
1378         }
1379
1380         /* the eol_style determines our first stop character and how many
1381          * characters we are going to drain afterwards. */
1382         switch (eol_style) {
1383         case EVBUFFER_EOL_ANY:
1384                 if (evbuffer_find_eol_char(&it) < 0)
1385                         goto done;
1386                 memcpy(&it2, &it, sizeof(it));
1387                 extra_drain = evbuffer_strspn(&it2, "\r\n");
1388                 break;
1389         case EVBUFFER_EOL_CRLF_STRICT: {
1390                 it = evbuffer_search(buffer, "\r\n", 2, &it);
1391                 if (it.pos < 0)
1392                         goto done;
1393                 extra_drain = 2;
1394                 break;
1395         }
1396         case EVBUFFER_EOL_CRLF:
1397                 while (1) {
1398                         if (evbuffer_find_eol_char(&it) < 0)
1399                                 goto done;
1400                         if (evbuffer_getchr(&it) == '\n') {
1401                                 extra_drain = 1;
1402                                 break;
1403                         } else if (!evbuffer_ptr_memcmp(
1404                                     buffer, &it, "\r\n", 2)) {
1405                                 extra_drain = 2;
1406                                 break;
1407                         } else {
1408                                 if (evbuffer_ptr_set(buffer, &it, 1,
1409                                         EVBUFFER_PTR_ADD)<0)
1410                                         goto done;
1411                         }
1412                 }
1413                 break;
1414         case EVBUFFER_EOL_LF:
1415                 if (evbuffer_strchr(&it, '\n') < 0)
1416                         goto done;
1417                 extra_drain = 1;
1418                 break;
1419         default:
1420                 goto done;
1421         }
1422
1423         ok = 1;
1424 done:
1425         EVBUFFER_UNLOCK(buffer);
1426
1427         if (!ok) {
1428                 it.pos = -1;
1429         }
1430         if (eol_len_out)
1431                 *eol_len_out = extra_drain;
1432
1433         return it;
1434 }
1435
1436 char *
1437 evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out,
1438                 enum evbuffer_eol_style eol_style)
1439 {
1440         struct evbuffer_ptr it;
1441         char *line;
1442         size_t n_to_copy=0, extra_drain=0;
1443         char *result = NULL;
1444
1445         EVBUFFER_LOCK(buffer);
1446
1447         if (buffer->freeze_start) {
1448                 goto done;
1449         }
1450
1451         it = evbuffer_search_eol(buffer, NULL, &extra_drain, eol_style);
1452         if (it.pos < 0)
1453                 goto done;
1454         n_to_copy = it.pos;
1455
1456         if ((line = mm_malloc(n_to_copy+1)) == NULL) {
1457                 event_warn("%s: out of memory", __func__);
1458                 goto done;
1459         }
1460
1461         evbuffer_remove(buffer, line, n_to_copy);
1462         line[n_to_copy] = '\0';
1463
1464         evbuffer_drain(buffer, extra_drain);
1465         result = line;
1466 done:
1467         EVBUFFER_UNLOCK(buffer);
1468
1469         if (n_read_out)
1470                 *n_read_out = result ? n_to_copy : 0;
1471
1472         return result;
1473 }
1474
1475 #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096
1476
1477 /* Adds data to an event buffer */
1478
1479 int
1480 evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
1481 {
1482         struct evbuffer_chain *chain, *tmp;
1483         const unsigned char *data = data_in;
1484         size_t remain, to_alloc;
1485         int result = -1;
1486
1487         EVBUFFER_LOCK(buf);
1488
1489         if (buf->freeze_end) {
1490                 goto done;
1491         }
1492
1493         chain = buf->last;
1494
1495         /* If there are no chains allocated for this buffer, allocate one
1496          * big enough to hold all the data. */
1497         if (chain == NULL) {
1498                 chain = evbuffer_chain_new(datlen);
1499                 if (!chain)
1500                         goto done;
1501                 evbuffer_chain_insert(buf, chain);
1502         }
1503
1504         if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1505                 remain = (size_t)(chain->buffer_len - chain->misalign - chain->off);
1506                 if (remain >= datlen) {
1507                         /* there's enough space to hold all the data in the
1508                          * current last chain */
1509                         memcpy(chain->buffer + chain->misalign + chain->off,
1510                             data, datlen);
1511                         chain->off += datlen;
1512                         buf->total_len += datlen;
1513                         buf->n_add_for_cb += datlen;
1514                         goto out;
1515                 } else if (!CHAIN_PINNED(chain) &&
1516                     evbuffer_chain_should_realign(chain, datlen)) {
1517                         /* we can fit the data into the misalignment */
1518                         evbuffer_chain_align(chain);
1519
1520                         memcpy(chain->buffer + chain->off, data, datlen);
1521                         chain->off += datlen;
1522                         buf->total_len += datlen;
1523                         buf->n_add_for_cb += datlen;
1524                         goto out;
1525                 }
1526         } else {
1527                 /* we cannot write any data to the last chain */
1528                 remain = 0;
1529         }
1530
1531         /* we need to add another chain */
1532         to_alloc = chain->buffer_len;
1533         if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2)
1534                 to_alloc <<= 1;
1535         if (datlen > to_alloc)
1536                 to_alloc = datlen;
1537         tmp = evbuffer_chain_new(to_alloc);
1538         if (tmp == NULL)
1539                 goto done;
1540
1541         if (remain) {
1542                 memcpy(chain->buffer + chain->misalign + chain->off,
1543                     data, remain);
1544                 chain->off += remain;
1545                 buf->total_len += remain;
1546                 buf->n_add_for_cb += remain;
1547         }
1548
1549         data += remain;
1550         datlen -= remain;
1551
1552         memcpy(tmp->buffer, data, datlen);
1553         tmp->off = datlen;
1554         evbuffer_chain_insert(buf, tmp);
1555
1556 out:
1557         evbuffer_invoke_callbacks(buf);
1558         result = 0;
1559 done:
1560         EVBUFFER_UNLOCK(buf);
1561         return result;
1562 }
1563
1564 int
1565 evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen)
1566 {
1567         struct evbuffer_chain *chain, *tmp;
1568         int result = -1;
1569
1570         EVBUFFER_LOCK(buf);
1571
1572         if (buf->freeze_start) {
1573                 goto done;
1574         }
1575
1576         chain = buf->first;
1577
1578         if (chain == NULL) {
1579                 chain = evbuffer_chain_new(datlen);
1580                 if (!chain)
1581                         goto done;
1582                 evbuffer_chain_insert(buf, chain);
1583         }
1584
1585         /* we cannot touch immutable buffers */
1586         if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1587                 /* If this chain is empty, we can treat it as
1588                  * 'empty at the beginning' rather than 'empty at the end' */
1589                 if (chain->off == 0)
1590                         chain->misalign = chain->buffer_len;
1591
1592                 if ((size_t)chain->misalign >= datlen) {
1593                         /* we have enough space to fit everything */
1594                         memcpy(chain->buffer + chain->misalign - datlen,
1595                             data, datlen);
1596                         chain->off += datlen;
1597                         chain->misalign -= datlen;
1598                         buf->total_len += datlen;
1599                         buf->n_add_for_cb += datlen;
1600                         goto out;
1601                 } else if (chain->misalign) {
1602                         /* we can only fit some of the data. */
1603                         memcpy(chain->buffer,
1604                             (char*)data + datlen - chain->misalign,
1605                             (size_t)chain->misalign);
1606                         chain->off += (size_t)chain->misalign;
1607                         buf->total_len += (size_t)chain->misalign;
1608                         buf->n_add_for_cb += (size_t)chain->misalign;
1609                         datlen -= (size_t)chain->misalign;
1610                         chain->misalign = 0;
1611                 }
1612         }
1613
1614         /* we need to add another chain */
1615         if ((tmp = evbuffer_chain_new(datlen)) == NULL)
1616                 goto done;
1617         buf->first = tmp;
1618         if (buf->last_with_datap == &buf->first)
1619                 buf->last_with_datap = &tmp->next;
1620
1621         tmp->next = chain;
1622
1623         tmp->off = datlen;
1624         tmp->misalign = tmp->buffer_len - datlen;
1625
1626         memcpy(tmp->buffer + tmp->misalign, data, datlen);
1627         buf->total_len += datlen;
1628         buf->n_add_for_cb += (size_t)chain->misalign;
1629
1630 out:
1631         evbuffer_invoke_callbacks(buf);
1632         result = 0;
1633 done:
1634         EVBUFFER_UNLOCK(buf);
1635         return result;
1636 }
1637
1638 /** Helper: realigns the memory in chain->buffer so that misalign is 0. */
1639 static void
1640 evbuffer_chain_align(struct evbuffer_chain *chain)
1641 {
1642         EVUTIL_ASSERT(!(chain->flags & EVBUFFER_IMMUTABLE));
1643         EVUTIL_ASSERT(!(chain->flags & EVBUFFER_MEM_PINNED_ANY));
1644         memmove(chain->buffer, chain->buffer + chain->misalign, chain->off);
1645         chain->misalign = 0;
1646 }
1647
1648 #define MAX_TO_COPY_IN_EXPAND 4096
1649 #define MAX_TO_REALIGN_IN_EXPAND 2048
1650
1651 /** Helper: return true iff we should realign chain to fit datalen bytes of
1652     data in it. */
1653 static int
1654 evbuffer_chain_should_realign(struct evbuffer_chain *chain,
1655     size_t datlen)
1656 {
1657         return chain->buffer_len - chain->off >= datlen &&
1658             (chain->off < chain->buffer_len / 2) &&
1659             (chain->off <= MAX_TO_REALIGN_IN_EXPAND);
1660 }
1661
1662 /* Expands the available space in the event buffer to at least datlen, all in
1663  * a single chunk.  Return that chunk. */
1664 static struct evbuffer_chain *
1665 evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen)
1666 {
1667         struct evbuffer_chain *chain, **chainp;
1668         struct evbuffer_chain *result = NULL;
1669         ASSERT_EVBUFFER_LOCKED(buf);
1670
1671         chainp = buf->last_with_datap;
1672
1673         /* XXX If *chainp is no longer writeable, but has enough space in its
1674          * misalign, this might be a bad idea: we could still use *chainp, not
1675          * (*chainp)->next. */
1676         if (*chainp && CHAIN_SPACE_LEN(*chainp) == 0)
1677                 chainp = &(*chainp)->next;
1678
1679         /* 'chain' now points to the first chain with writable space (if any)
1680          * We will either use it, realign it, replace it, or resize it. */
1681         chain = *chainp;
1682
1683         if (chain == NULL ||
1684             (chain->flags & (EVBUFFER_IMMUTABLE|EVBUFFER_MEM_PINNED_ANY))) {
1685                 /* We can't use the last_with_data chain at all.  Just add a
1686                  * new one that's big enough. */
1687                 goto insert_new;
1688         }
1689
1690         /* If we can fit all the data, then we don't have to do anything */
1691         if (CHAIN_SPACE_LEN(chain) >= datlen) {
1692                 result = chain;
1693                 goto ok;
1694         }
1695
1696         /* If the chain is completely empty, just replace it by adding a new
1697          * empty chain. */
1698         if (chain->off == 0) {
1699                 goto insert_new;
1700         }
1701
1702         /* If the misalignment plus the remaining space fulfills our data
1703          * needs, we could just force an alignment to happen.  Afterwards, we
1704          * have enough space.  But only do this if we're saving a lot of space
1705          * and not moving too much data.  Otherwise the space savings are
1706          * probably offset by the time lost in copying.
1707          */
1708         if (evbuffer_chain_should_realign(chain, datlen)) {
1709                 evbuffer_chain_align(chain);
1710                 result = chain;
1711                 goto ok;
1712         }
1713
1714         /* At this point, we can either resize the last chunk with space in
1715          * it, use the next chunk after it, or   If we add a new chunk, we waste
1716          * CHAIN_SPACE_LEN(chain) bytes in the former last chunk.  If we
1717          * resize, we have to copy chain->off bytes.
1718          */
1719
1720         /* Would expanding this chunk be affordable and worthwhile? */
1721         if (CHAIN_SPACE_LEN(chain) < chain->buffer_len / 8 ||
1722             chain->off > MAX_TO_COPY_IN_EXPAND) {
1723                 /* It's not worth resizing this chain. Can the next one be
1724                  * used? */
1725                 if (chain->next && CHAIN_SPACE_LEN(chain->next) >= datlen) {
1726                         /* Yes, we can just use the next chain (which should
1727                          * be empty. */
1728                         result = chain->next;
1729                         goto ok;
1730                 } else {
1731                         /* No; append a new chain (which will free all
1732                          * terminal empty chains.) */
1733                         goto insert_new;
1734                 }
1735         } else {
1736                 /* Okay, we're going to try to resize this chain: Not doing so
1737                  * would waste at least 1/8 of its current allocation, and we
1738                  * can do so without having to copy more than
1739                  * MAX_TO_COPY_IN_EXPAND bytes. */
1740                 /* figure out how much space we need */
1741                 size_t length = chain->off + datlen;
1742                 struct evbuffer_chain *tmp = evbuffer_chain_new(length);
1743                 if (tmp == NULL)
1744                         goto err;
1745
1746                 /* copy the data over that we had so far */
1747                 tmp->off = chain->off;
1748                 memcpy(tmp->buffer, chain->buffer + chain->misalign,
1749                     chain->off);
1750                 /* fix up the list */
1751                 EVUTIL_ASSERT(*chainp == chain);
1752                 result = *chainp = tmp;
1753
1754                 if (buf->last == chain)
1755                         buf->last = tmp;
1756
1757                 tmp->next = chain->next;
1758                 evbuffer_chain_free(chain);
1759                 goto ok;
1760         }
1761
1762 insert_new:
1763         result = evbuffer_chain_insert_new(buf, datlen);
1764         if (!result)
1765                 goto err;
1766 ok:
1767         EVUTIL_ASSERT(result);
1768         EVUTIL_ASSERT(CHAIN_SPACE_LEN(result) >= datlen);
1769 err:
1770         return result;
1771 }
1772
1773 /* Make sure that datlen bytes are available for writing in the last n
1774  * chains.  Never copies or moves data. */
1775 int
1776 _evbuffer_expand_fast(struct evbuffer *buf, size_t datlen, int n)
1777 {
1778         struct evbuffer_chain *chain = buf->last, *tmp, *next;
1779         size_t avail;
1780         int used;
1781
1782         ASSERT_EVBUFFER_LOCKED(buf);
1783         EVUTIL_ASSERT(n >= 2);
1784
1785         if (chain == NULL || (chain->flags & EVBUFFER_IMMUTABLE)) {
1786                 /* There is no last chunk, or we can't touch the last chunk.
1787                  * Just add a new chunk. */
1788                 chain = evbuffer_chain_new(datlen);
1789                 if (chain == NULL)
1790                         return (-1);
1791
1792                 evbuffer_chain_insert(buf, chain);
1793                 return (0);
1794         }
1795
1796         used = 0; /* number of chains we're using space in. */
1797         avail = 0; /* how much space they have. */
1798         /* How many bytes can we stick at the end of buffer as it is?  Iterate
1799          * over the chains at the end of the buffer, tring to see how much
1800          * space we have in the first n. */
1801         for (chain = *buf->last_with_datap; chain; chain = chain->next) {
1802                 if (chain->off) {
1803                         size_t space = (size_t) CHAIN_SPACE_LEN(chain);
1804                         EVUTIL_ASSERT(chain == *buf->last_with_datap);
1805                         if (space) {
1806                                 avail += space;
1807                                 ++used;
1808                         }
1809                 } else {
1810                         /* No data in chain; realign it. */
1811                         chain->misalign = 0;
1812                         avail += chain->buffer_len;
1813                         ++used;
1814                 }
1815                 if (avail >= datlen) {
1816                         /* There is already enough space.  Just return */
1817                         return (0);
1818                 }
1819                 if (used == n)
1820                         break;
1821         }
1822
1823         /* There wasn't enough space in the first n chains with space in
1824          * them. Either add a new chain with enough space, or replace all
1825          * empty chains with one that has enough space, depending on n. */
1826         if (used < n) {
1827                 /* The loop ran off the end of the chains before it hit n
1828                  * chains; we can add another. */
1829                 EVUTIL_ASSERT(chain == NULL);
1830
1831                 tmp = evbuffer_chain_new(datlen - avail);
1832                 if (tmp == NULL)
1833                         return (-1);
1834
1835                 buf->last->next = tmp;
1836                 buf->last = tmp;
1837                 /* (we would only set last_with_data if we added the first
1838                  * chain. But if the buffer had no chains, we would have
1839                  * just allocated a new chain earlier) */
1840                 return (0);
1841         } else {
1842                 /* Nuke _all_ the empty chains. */
1843                 int rmv_all = 0; /* True iff we removed last_with_data. */
1844                 chain = *buf->last_with_datap;
1845                 if (!chain->off) {
1846                         EVUTIL_ASSERT(chain == buf->first);
1847                         rmv_all = 1;
1848                         avail = 0;
1849                 } else {
1850                         avail = (size_t) CHAIN_SPACE_LEN(chain);
1851                         chain = chain->next;
1852                 }
1853
1854
1855                 for (; chain; chain = next) {
1856                         next = chain->next;
1857                         EVUTIL_ASSERT(chain->off == 0);
1858                         evbuffer_chain_free(chain);
1859                 }
1860                 tmp = evbuffer_chain_new(datlen - avail);
1861                 if (tmp == NULL) {
1862                         if (rmv_all) {
1863                                 ZERO_CHAIN(buf);
1864                         } else {
1865                                 buf->last = *buf->last_with_datap;
1866                                 (*buf->last_with_datap)->next = NULL;
1867                         }
1868                         return (-1);
1869                 }
1870
1871                 if (rmv_all) {
1872                         buf->first = buf->last = tmp;
1873                         buf->last_with_datap = &buf->first;
1874                 } else {
1875                         (*buf->last_with_datap)->next = tmp;
1876                         buf->last = tmp;
1877                 }
1878                 return (0);
1879         }
1880 }
1881
1882 int
1883 evbuffer_expand(struct evbuffer *buf, size_t datlen)
1884 {
1885         struct evbuffer_chain *chain;
1886
1887         EVBUFFER_LOCK(buf);
1888         chain = evbuffer_expand_singlechain(buf, datlen);
1889         EVBUFFER_UNLOCK(buf);
1890         return chain ? 0 : -1;
1891 }
1892
1893 /*
1894  * Reads data from a file descriptor into a buffer.
1895  */
1896
1897 #if defined(_EVENT_HAVE_SYS_UIO_H) || defined(WIN32)
1898 #define USE_IOVEC_IMPL
1899 #endif
1900
1901 #ifdef USE_IOVEC_IMPL
1902
1903 #ifdef _EVENT_HAVE_SYS_UIO_H
1904 /* number of iovec we use for writev, fragmentation is going to determine
1905  * how much we end up writing */
1906
1907 #define DEFAULT_WRITE_IOVEC 128
1908
1909 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
1910 #define NUM_WRITE_IOVEC UIO_MAXIOV
1911 #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
1912 #define NUM_WRITE_IOVEC IOV_MAX
1913 #else
1914 #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
1915 #endif
1916
1917 #define IOV_TYPE struct iovec
1918 #define IOV_PTR_FIELD iov_base
1919 #define IOV_LEN_FIELD iov_len
1920 #define IOV_LEN_TYPE size_t
1921 #else
1922 #define NUM_WRITE_IOVEC 16
1923 #define IOV_TYPE WSABUF
1924 #define IOV_PTR_FIELD buf
1925 #define IOV_LEN_FIELD len
1926 #define IOV_LEN_TYPE unsigned long
1927 #endif
1928 #endif
1929 #define NUM_READ_IOVEC 4
1930
1931 #define EVBUFFER_MAX_READ       4096
1932
1933 /** Helper function to figure out which space to use for reading data into
1934     an evbuffer.  Internal use only.
1935
1936     @param buf The buffer to read into
1937     @param howmuch How much we want to read.
1938     @param vecs An array of two or more iovecs or WSABUFs.
1939     @param n_vecs_avail The length of vecs
1940     @param chainp A pointer to a variable to hold the first chain we're
1941       reading into.
1942     @param exact Boolean: if true, we do not provide more than 'howmuch'
1943       space in the vectors, even if more space is available.
1944     @return The number of buffers we're using.
1945  */
1946 int
1947 _evbuffer_read_setup_vecs(struct evbuffer *buf, ev_ssize_t howmuch,
1948     struct evbuffer_iovec *vecs, int n_vecs_avail,
1949     struct evbuffer_chain ***chainp, int exact)
1950 {
1951         struct evbuffer_chain *chain;
1952         struct evbuffer_chain **firstchainp;
1953         size_t so_far;
1954         int i;
1955         ASSERT_EVBUFFER_LOCKED(buf);
1956
1957         if (howmuch < 0)
1958                 return -1;
1959
1960         so_far = 0;
1961         /* Let firstchain be the first chain with any space on it */
1962         firstchainp = buf->last_with_datap;
1963         if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
1964                 firstchainp = &(*firstchainp)->next;
1965         }
1966
1967         chain = *firstchainp;
1968         for (i = 0; i < n_vecs_avail && so_far < (size_t)howmuch; ++i) {
1969                 size_t avail = (size_t) CHAIN_SPACE_LEN(chain);
1970                 if (avail > (howmuch - so_far) && exact)
1971                         avail = howmuch - so_far;
1972                 vecs[i].iov_base = CHAIN_SPACE_PTR(chain);
1973                 vecs[i].iov_len = avail;
1974                 so_far += avail;
1975                 chain = chain->next;
1976         }
1977
1978         *chainp = firstchainp;
1979         return i;
1980 }
1981
1982 static int
1983 get_n_bytes_readable_on_socket(evutil_socket_t fd)
1984 {
1985 #if defined(FIONREAD) && defined(WIN32)
1986         unsigned long lng = EVBUFFER_MAX_READ;
1987         if (ioctlsocket(fd, FIONREAD, &lng) < 0)
1988                 return -1;
1989         return (int)lng;
1990 #elif defined(FIONREAD)
1991         int n = EVBUFFER_MAX_READ;
1992         if (ioctl(fd, FIONREAD, &n) < 0)
1993                 return -1;
1994         return n;
1995 #else
1996         return EVBUFFER_MAX_READ;
1997 #endif
1998 }
1999
2000 /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
2001  * as howmuch? */
2002 int
2003 evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch)
2004 {
2005         struct evbuffer_chain **chainp;
2006         int n;
2007         int result;
2008
2009 #ifdef USE_IOVEC_IMPL
2010         int nvecs, i, remaining;
2011 #else
2012         struct evbuffer_chain *chain;
2013         unsigned char *p;
2014 #endif
2015
2016         EVBUFFER_LOCK(buf);
2017
2018         if (buf->freeze_end) {
2019                 result = -1;
2020                 goto done;
2021         }
2022
2023         n = get_n_bytes_readable_on_socket(fd);
2024         if (n <= 0 || n > EVBUFFER_MAX_READ)
2025                 n = EVBUFFER_MAX_READ;
2026         if (howmuch < 0 || howmuch > n)
2027                 howmuch = n;
2028
2029 #ifdef USE_IOVEC_IMPL
2030         /* Since we can use iovecs, we're willing to use the last
2031          * NUM_READ_IOVEC chains. */
2032         if (_evbuffer_expand_fast(buf, howmuch, NUM_READ_IOVEC) == -1) {
2033                 result = -1;
2034                 goto done;
2035         } else {
2036                 IOV_TYPE vecs[NUM_READ_IOVEC];
2037 #ifdef _EVBUFFER_IOVEC_IS_NATIVE
2038                 nvecs = _evbuffer_read_setup_vecs(buf, howmuch, vecs,
2039                     NUM_READ_IOVEC, &chainp, 1);
2040 #else
2041                 /* We aren't using the native struct iovec.  Therefore,
2042                    we are on win32. */
2043                 struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC];
2044                 nvecs = _evbuffer_read_setup_vecs(buf, howmuch, ev_vecs, 2,
2045                     &chainp, 1);
2046
2047                 for (i=0; i < nvecs; ++i)
2048                         WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]);
2049 #endif
2050
2051 #ifdef WIN32
2052                 {
2053                         DWORD bytesRead;
2054                         DWORD flags=0;
2055                         if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) {
2056                                 /* The read failed. It might be a close,
2057                                  * or it might be an error. */
2058                                 if (WSAGetLastError() == WSAECONNABORTED)
2059                                         n = 0;
2060                                 else
2061                                         n = -1;
2062                         } else
2063                                 n = bytesRead;
2064                 }
2065 #else
2066                 n = readv(fd, vecs, nvecs);
2067 #endif
2068         }
2069
2070 #else /*!USE_IOVEC_IMPL*/
2071         /* If we don't have FIONREAD, we might waste some space here */
2072         /* XXX we _will_ waste some space here if there is any space left
2073          * over on buf->last. */
2074         if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) {
2075                 result = -1;
2076                 goto done;
2077         }
2078
2079         /* We can append new data at this point */
2080         p = chain->buffer + chain->misalign + chain->off;
2081
2082 #ifndef WIN32
2083         n = read(fd, p, howmuch);
2084 #else
2085         n = recv(fd, p, howmuch, 0);
2086 #endif
2087 #endif /* USE_IOVEC_IMPL */
2088
2089         if (n == -1) {
2090                 result = -1;
2091                 goto done;
2092         }
2093         if (n == 0) {
2094                 result = 0;
2095                 goto done;
2096         }
2097
2098 #ifdef USE_IOVEC_IMPL
2099         remaining = n;
2100         for (i=0; i < nvecs; ++i) {
2101                 ev_ssize_t space = (ev_ssize_t) CHAIN_SPACE_LEN(*chainp);
2102                 if (space < remaining) {
2103                         (*chainp)->off += space;
2104                         remaining -= (int)space;
2105                 } else {
2106                         (*chainp)->off += remaining;
2107                         buf->last_with_datap = chainp;
2108                         break;
2109                 }
2110                 chainp = &(*chainp)->next;
2111         }
2112 #else
2113         chain->off += n;
2114         advance_last_with_data(buf);
2115 #endif
2116         buf->total_len += n;
2117         buf->n_add_for_cb += n;
2118
2119         /* Tell someone about changes in this buffer */
2120         evbuffer_invoke_callbacks(buf);
2121         result = n;
2122 done:
2123         EVBUFFER_UNLOCK(buf);
2124         return result;
2125 }
2126
2127 #ifdef WIN32
2128 static int
2129 evbuffer_readfile(struct evbuffer *buf, evutil_socket_t fd, ev_ssize_t howmuch)
2130 {
2131         int result;
2132         int nchains, n;
2133         struct evbuffer_iovec v[2];
2134
2135         EVBUFFER_LOCK(buf);
2136
2137         if (buf->freeze_end) {
2138                 result = -1;
2139                 goto done;
2140         }
2141
2142         if (howmuch < 0)
2143                 howmuch = 16384;
2144
2145
2146         /* XXX we _will_ waste some space here if there is any space left
2147          * over on buf->last. */
2148         nchains = evbuffer_reserve_space(buf, howmuch, v, 2);
2149         if (nchains < 1 || nchains > 2) {
2150                 result = -1;
2151                 goto done;
2152         }
2153         n = read((int)fd, v[0].iov_base, (unsigned int)v[0].iov_len);
2154         if (n <= 0) {
2155                 result = n;
2156                 goto done;
2157         }
2158         v[0].iov_len = (IOV_LEN_TYPE) n; /* XXXX another problem with big n.*/
2159         if (nchains > 1) {
2160                 n = read((int)fd, v[1].iov_base, (unsigned int)v[1].iov_len);
2161                 if (n <= 0) {
2162                         result = (unsigned long) v[0].iov_len;
2163                         evbuffer_commit_space(buf, v, 1);
2164                         goto done;
2165                 }
2166                 v[1].iov_len = n;
2167         }
2168         evbuffer_commit_space(buf, v, nchains);
2169
2170         result = n;
2171 done:
2172         EVBUFFER_UNLOCK(buf);
2173         return result;
2174 }
2175 #endif
2176
2177 #ifdef USE_IOVEC_IMPL
2178 static inline int
2179 evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd,
2180     ev_ssize_t howmuch)
2181 {
2182         IOV_TYPE iov[NUM_WRITE_IOVEC];
2183         struct evbuffer_chain *chain = buffer->first;
2184         int n, i = 0;
2185
2186         if (howmuch < 0)
2187                 return -1;
2188
2189         ASSERT_EVBUFFER_LOCKED(buffer);
2190         /* XXX make this top out at some maximal data length?  if the
2191          * buffer has (say) 1MB in it, split over 128 chains, there's
2192          * no way it all gets written in one go. */
2193         while (chain != NULL && i < NUM_WRITE_IOVEC && howmuch) {
2194 #ifdef USE_SENDFILE
2195                 /* we cannot write the file info via writev */
2196                 if (chain->flags & EVBUFFER_SENDFILE)
2197                         break;
2198 #endif
2199                 iov[i].IOV_PTR_FIELD = (void *) (chain->buffer + chain->misalign);
2200                 if ((size_t)howmuch >= chain->off) {
2201                         /* XXXcould be problematic when windows supports mmap*/
2202                         iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)chain->off;
2203                         howmuch -= chain->off;
2204                 } else {
2205                         /* XXXcould be problematic when windows supports mmap*/
2206                         iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)howmuch;
2207                         break;
2208                 }
2209                 chain = chain->next;
2210         }
2211 #ifdef WIN32
2212         {
2213                 DWORD bytesSent;
2214                 if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL))
2215                         n = -1;
2216                 else
2217                         n = bytesSent;
2218         }
2219 #else
2220         n = writev(fd, iov, i);
2221 #endif
2222         return (n);
2223 }
2224 #endif
2225
2226 #ifdef USE_SENDFILE
2227 static inline int
2228 evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t fd,
2229     ev_ssize_t howmuch)
2230 {
2231         struct evbuffer_chain *chain = buffer->first;
2232         struct evbuffer_chain_fd *info =
2233             EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain);
2234 #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
2235         int res;
2236         off_t len = chain->off;
2237 #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
2238         ev_ssize_t res;
2239         off_t offset = chain->misalign;
2240 #endif
2241
2242         ASSERT_EVBUFFER_LOCKED(buffer);
2243
2244 #if defined(SENDFILE_IS_MACOSX)
2245         res = sendfile(info->fd, fd, chain->misalign, &len, NULL, 0);
2246         if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2247                 return (-1);
2248
2249         return (len);
2250 #elif defined(SENDFILE_IS_FREEBSD)
2251         res = sendfile(info->fd, fd, chain->misalign, chain->off, NULL, &len, 0);
2252         if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2253                 return (-1);
2254
2255         return (len);
2256 #elif defined(SENDFILE_IS_LINUX)
2257         /* TODO(niels): implement splice */
2258         res = sendfile(fd, info->fd, &offset, chain->off);
2259         if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2260                 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2261                 return (0);
2262         }
2263         return (res);
2264 #elif defined(SENDFILE_IS_SOLARIS)
2265         res = sendfile(fd, info->fd, &offset, chain->off);
2266         if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2267                 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2268                 return (0);
2269         }
2270         return (res);
2271 #endif
2272 }
2273 #endif
2274
2275 int
2276 evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd,
2277     ev_ssize_t howmuch)
2278 {
2279         int n = -1;
2280
2281         EVBUFFER_LOCK(buffer);
2282
2283         if (buffer->freeze_start) {
2284                 goto done;
2285         }
2286
2287         if (howmuch < 0 || (size_t)howmuch > buffer->total_len)
2288                 howmuch = buffer->total_len;
2289
2290         if (howmuch > 0) {
2291 #ifdef USE_SENDFILE
2292                 struct evbuffer_chain *chain = buffer->first;
2293                 if (chain != NULL && (chain->flags & EVBUFFER_SENDFILE))
2294                         n = evbuffer_write_sendfile(buffer, fd, howmuch);
2295                 else {
2296 #endif
2297 #ifdef USE_IOVEC_IMPL
2298                 n = evbuffer_write_iovec(buffer, fd, howmuch);
2299 #elif defined(WIN32)
2300                 /* XXX(nickm) Don't disable this code until we know if
2301                  * the WSARecv code above works. */
2302                 void *p = evbuffer_pullup(buffer, howmuch);
2303                 n = send(fd, p, howmuch, 0);
2304 #else
2305                 void *p = evbuffer_pullup(buffer, howmuch);
2306                 n = write(fd, p, howmuch);
2307 #endif
2308 #ifdef USE_SENDFILE
2309                 }
2310 #endif
2311         }
2312
2313         if (n > 0)
2314                 evbuffer_drain(buffer, n);
2315
2316 done:
2317         EVBUFFER_UNLOCK(buffer);
2318         return (n);
2319 }
2320
2321 int
2322 evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd)
2323 {
2324         return evbuffer_write_atmost(buffer, fd, -1);
2325 }
2326
2327 unsigned char *
2328 evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len)
2329 {
2330         unsigned char *search;
2331         struct evbuffer_ptr ptr;
2332
2333         EVBUFFER_LOCK(buffer);
2334
2335         ptr = evbuffer_search(buffer, (const char *)what, len, NULL);
2336         if (ptr.pos < 0) {
2337                 search = NULL;
2338         } else {
2339                 search = evbuffer_pullup(buffer, ptr.pos + len);
2340                 if (search)
2341                         search += ptr.pos;
2342         }
2343         EVBUFFER_UNLOCK(buffer);
2344         return search;
2345 }
2346
2347 int
2348 evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos,
2349     size_t position, enum evbuffer_ptr_how how)
2350 {
2351         size_t left = position;
2352         struct evbuffer_chain *chain = NULL;
2353
2354         EVBUFFER_LOCK(buf);
2355
2356         switch (how) {
2357         case EVBUFFER_PTR_SET:
2358                 chain = buf->first;
2359                 pos->pos = position;
2360                 position = 0;
2361                 break;
2362         case EVBUFFER_PTR_ADD:
2363                 /* this avoids iterating over all previous chains if
2364                    we just want to advance the position */
2365                 chain = pos->_internal.chain;
2366                 pos->pos += position;
2367                 position = pos->_internal.pos_in_chain;
2368                 break;
2369         }
2370
2371         while (chain && position + left >= chain->off) {
2372                 left -= chain->off - position;
2373                 chain = chain->next;
2374                 position = 0;
2375         }
2376         if (chain) {
2377                 pos->_internal.chain = chain;
2378                 pos->_internal.pos_in_chain = position + left;
2379         } else {
2380                 pos->_internal.chain = NULL;
2381                 pos->pos = -1;
2382         }
2383
2384         EVBUFFER_UNLOCK(buf);
2385
2386         return chain != NULL ? 0 : -1;
2387 }
2388
2389 /**
2390    Compare the bytes in buf at position pos to the len bytes in mem.  Return
2391    less than 0, 0, or greater than 0 as memcmp.
2392  */
2393 static int
2394 evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos,
2395     const char *mem, size_t len)
2396 {
2397         struct evbuffer_chain *chain;
2398         size_t position;
2399         int r;
2400
2401         ASSERT_EVBUFFER_LOCKED(buf);
2402
2403         if (pos->pos + len > buf->total_len)
2404                 return -1;
2405
2406         chain = pos->_internal.chain;
2407         position = pos->_internal.pos_in_chain;
2408         while (len && chain) {
2409                 size_t n_comparable;
2410                 if (len + position > chain->off)
2411                         n_comparable = chain->off - position;
2412                 else
2413                         n_comparable = len;
2414                 r = memcmp(chain->buffer + chain->misalign + position, mem,
2415                     n_comparable);
2416                 if (r)
2417                         return r;
2418                 mem += n_comparable;
2419                 len -= n_comparable;
2420                 position = 0;
2421                 chain = chain->next;
2422         }
2423
2424         return 0;
2425 }
2426
2427 struct evbuffer_ptr
2428 evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start)
2429 {
2430         return evbuffer_search_range(buffer, what, len, start, NULL);
2431 }
2432
2433 struct evbuffer_ptr
2434 evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end)
2435 {
2436         struct evbuffer_ptr pos;
2437         struct evbuffer_chain *chain, *last_chain = NULL;
2438         const unsigned char *p;
2439         char first;
2440
2441         EVBUFFER_LOCK(buffer);
2442
2443         if (start) {
2444                 memcpy(&pos, start, sizeof(pos));
2445                 chain = pos._internal.chain;
2446         } else {
2447                 pos.pos = 0;
2448                 chain = pos._internal.chain = buffer->first;
2449                 pos._internal.pos_in_chain = 0;
2450         }
2451
2452         if (end)
2453                 last_chain = end->_internal.chain;
2454
2455         if (!len || len > EV_SSIZE_MAX)
2456                 goto done;
2457
2458         first = what[0];
2459
2460         while (chain) {
2461                 const unsigned char *start_at =
2462                     chain->buffer + chain->misalign +
2463                     pos._internal.pos_in_chain;
2464                 p = memchr(start_at, first,
2465                     chain->off - pos._internal.pos_in_chain);
2466                 if (p) {
2467                         pos.pos += p - start_at;
2468                         pos._internal.pos_in_chain += p - start_at;
2469                         if (!evbuffer_ptr_memcmp(buffer, &pos, what, len)) {
2470                                 if (end && pos.pos + (ev_ssize_t)len > end->pos)
2471                                         goto not_found;
2472                                 else
2473                                         goto done;
2474                         }
2475                         ++pos.pos;
2476                         ++pos._internal.pos_in_chain;
2477                         if (pos._internal.pos_in_chain == chain->off) {
2478                                 chain = pos._internal.chain = chain->next;
2479                                 pos._internal.pos_in_chain = 0;
2480                         }
2481                 } else {
2482                         if (chain == last_chain)
2483                                 goto not_found;
2484                         pos.pos += chain->off - pos._internal.pos_in_chain;
2485                         chain = pos._internal.chain = chain->next;
2486                         pos._internal.pos_in_chain = 0;
2487                 }
2488         }
2489
2490 not_found:
2491         pos.pos = -1;
2492         pos._internal.chain = NULL;
2493 done:
2494         EVBUFFER_UNLOCK(buffer);
2495         return pos;
2496 }
2497
2498 int
2499 evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len,
2500     struct evbuffer_ptr *start_at,
2501     struct evbuffer_iovec *vec, int n_vec)
2502 {
2503         struct evbuffer_chain *chain;
2504         int idx = 0;
2505         ev_ssize_t len_so_far = 0;
2506
2507         EVBUFFER_LOCK(buffer);
2508
2509         if (start_at) {
2510                 chain = start_at->_internal.chain;
2511                 len_so_far = chain->off
2512                     - start_at->_internal.pos_in_chain;
2513                 idx = 1;
2514                 if (n_vec > 0) {
2515                         vec[0].iov_base = chain->buffer + chain->misalign
2516                             + start_at->_internal.pos_in_chain;
2517                         vec[0].iov_len = len_so_far;
2518                 }
2519                 chain = chain->next;
2520         } else {
2521                 chain = buffer->first;
2522         }
2523
2524         while (chain) {
2525                 if (len >= 0 && len_so_far >= len)
2526                         break;
2527                 if (idx<n_vec) {
2528                         vec[idx].iov_base = chain->buffer + chain->misalign;
2529                         vec[idx].iov_len = chain->off;
2530                 } else if (len<0)
2531                         break;
2532                 ++idx;
2533                 len_so_far += chain->off;
2534                 chain = chain->next;
2535         }
2536
2537         EVBUFFER_UNLOCK(buffer);
2538
2539         return idx;
2540 }
2541
2542
2543 int
2544 evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap)
2545 {
2546         char *buffer;
2547         size_t space;
2548         int sz, result = -1;
2549         va_list aq;
2550         struct evbuffer_chain *chain;
2551
2552
2553         EVBUFFER_LOCK(buf);
2554
2555         if (buf->freeze_end) {
2556                 goto done;
2557         }
2558
2559         /* make sure that at least some space is available */
2560         if ((chain = evbuffer_expand_singlechain(buf, 64)) == NULL)
2561                 goto done;
2562
2563         for (;;) {
2564 #if 0
2565                 size_t used = chain->misalign + chain->off;
2566                 buffer = (char *)chain->buffer + chain->misalign + chain->off;
2567                 EVUTIL_ASSERT(chain->buffer_len >= used);
2568                 space = chain->buffer_len - used;
2569 #endif
2570                 buffer = (char*) CHAIN_SPACE_PTR(chain);
2571                 space = (size_t) CHAIN_SPACE_LEN(chain);
2572
2573 #ifndef va_copy
2574 #define va_copy(dst, src)       memcpy(&(dst), &(src), sizeof(va_list))
2575 #endif
2576                 va_copy(aq, ap);
2577
2578                 sz = evutil_vsnprintf(buffer, space, fmt, aq);
2579
2580                 va_end(aq);
2581
2582                 if (sz < 0)
2583                         goto done;
2584                 if ((size_t)sz < space) {
2585                         chain->off += sz;
2586                         buf->total_len += sz;
2587                         buf->n_add_for_cb += sz;
2588
2589                         advance_last_with_data(buf);
2590                         evbuffer_invoke_callbacks(buf);
2591                         result = sz;
2592                         goto done;
2593                 }
2594                 if ((chain = evbuffer_expand_singlechain(buf, sz + 1)) == NULL)
2595                         goto done;
2596         }
2597         /* NOTREACHED */
2598
2599 done:
2600         EVBUFFER_UNLOCK(buf);
2601         return result;
2602 }
2603
2604 int
2605 evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...)
2606 {
2607         int res = -1;
2608         va_list ap;
2609
2610         va_start(ap, fmt);
2611         res = evbuffer_add_vprintf(buf, fmt, ap);
2612         va_end(ap);
2613
2614         return (res);
2615 }
2616
2617 int
2618 evbuffer_add_reference(struct evbuffer *outbuf,
2619     const void *data, size_t datlen,
2620     evbuffer_ref_cleanup_cb cleanupfn, void *extra)
2621 {
2622         struct evbuffer_chain *chain;
2623         struct evbuffer_chain_reference *info;
2624         int result = -1;
2625
2626         chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_reference));
2627         if (!chain)
2628                 return (-1);
2629         chain->flags |= EVBUFFER_REFERENCE | EVBUFFER_IMMUTABLE;
2630         chain->buffer = (u_char *)data;
2631         chain->buffer_len = datlen;
2632         chain->off = datlen;
2633
2634         info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference, chain);
2635         info->cleanupfn = cleanupfn;
2636         info->extra = extra;
2637
2638         EVBUFFER_LOCK(outbuf);
2639         if (outbuf->freeze_end) {
2640                 /* don't call chain_free; we do not want to actually invoke
2641                  * the cleanup function */
2642                 mm_free(chain);
2643                 goto done;
2644         }
2645         evbuffer_chain_insert(outbuf, chain);
2646         outbuf->n_add_for_cb += datlen;
2647
2648         evbuffer_invoke_callbacks(outbuf);
2649
2650         result = 0;
2651 done:
2652         EVBUFFER_UNLOCK(outbuf);
2653
2654         return result;
2655 }
2656
2657 /* TODO(niels): maybe we don't want to own the fd, however, in that
2658  * case, we should dup it - dup is cheap.  Perhaps, we should use a
2659  * callback instead?
2660  */
2661 /* TODO(niels): we may want to add to automagically convert to mmap, in
2662  * case evbuffer_remove() or evbuffer_pullup() are being used.
2663  */
2664 int
2665 evbuffer_add_file(struct evbuffer *outbuf, int fd,
2666     ev_off_t offset, ev_off_t length)
2667 {
2668 #if defined(USE_SENDFILE) || defined(_EVENT_HAVE_MMAP)
2669         struct evbuffer_chain *chain;
2670         struct evbuffer_chain_fd *info;
2671 #endif
2672         int ok = 1;
2673
2674 #if defined(USE_SENDFILE)
2675         if (use_sendfile) {
2676                 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_fd));
2677                 if (chain == NULL) {
2678                         event_warn("%s: out of memory", __func__);
2679                         return (-1);
2680                 }
2681
2682                 chain->flags |= EVBUFFER_SENDFILE | EVBUFFER_IMMUTABLE;
2683                 chain->buffer = NULL;   /* no reading possible */
2684                 chain->buffer_len = length + offset;
2685                 chain->off = length;
2686                 chain->misalign = offset;
2687
2688                 info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain);
2689                 info->fd = fd;
2690
2691                 EVBUFFER_LOCK(outbuf);
2692                 if (outbuf->freeze_end) {
2693                         mm_free(chain);
2694                         ok = 0;
2695                 } else {
2696                         outbuf->n_add_for_cb += length;
2697                         evbuffer_chain_insert(outbuf, chain);
2698                 }
2699         } else
2700 #endif
2701 #if defined(_EVENT_HAVE_MMAP)
2702         if (use_mmap) {
2703                 void *mapped = mmap(NULL, length + offset, PROT_READ,
2704 #ifdef MAP_NOCACHE
2705                     MAP_NOCACHE |
2706 #endif
2707 #ifdef MAP_FILE
2708                     MAP_FILE |
2709 #endif
2710                     MAP_PRIVATE,
2711                     fd, 0);
2712                 /* some mmap implementations require offset to be a multiple of
2713                  * the page size.  most users of this api, are likely to use 0
2714                  * so mapping everything is not likely to be a problem.
2715                  * TODO(niels): determine page size and round offset to that
2716                  * page size to avoid mapping too much memory.
2717                  */
2718                 if (mapped == MAP_FAILED) {
2719                         event_warn("%s: mmap(%d, %d, %zu) failed",
2720                             __func__, fd, 0, (size_t)(offset + length));
2721                         return (-1);
2722                 }
2723                 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_fd));
2724                 if (chain == NULL) {
2725                         event_warn("%s: out of memory", __func__);
2726                         munmap(mapped, length);
2727                         return (-1);
2728                 }
2729
2730                 chain->flags |= EVBUFFER_MMAP | EVBUFFER_IMMUTABLE;
2731                 chain->buffer = mapped;
2732                 chain->buffer_len = length + offset;
2733                 chain->off = length + offset;
2734
2735                 info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd, chain);
2736                 info->fd = fd;
2737
2738                 EVBUFFER_LOCK(outbuf);
2739                 if (outbuf->freeze_end) {
2740                         info->fd = -1;
2741                         evbuffer_chain_free(chain);
2742                         ok = 0;
2743                 } else {
2744                         outbuf->n_add_for_cb += length;
2745
2746                         evbuffer_chain_insert(outbuf, chain);
2747
2748                         /* we need to subtract whatever we don't need */
2749                         evbuffer_drain(outbuf, offset);
2750                 }
2751         } else
2752 #endif
2753         {
2754                 /* the default implementation */
2755                 struct evbuffer *tmp = evbuffer_new();
2756                 ev_ssize_t read;
2757
2758                 if (tmp == NULL)
2759                         return (-1);
2760
2761 #ifdef WIN32
2762 #define lseek _lseeki64
2763 #endif
2764                 if (lseek(fd, offset, SEEK_SET) == -1) {
2765                         evbuffer_free(tmp);
2766                         return (-1);
2767                 }
2768
2769                 /* we add everything to a temporary buffer, so that we
2770                  * can abort without side effects if the read fails.
2771                  */
2772                 while (length) {
2773                         read = evbuffer_readfile(tmp, fd, (ev_ssize_t)length);
2774                         if (read == -1) {
2775                                 evbuffer_free(tmp);
2776                                 return (-1);
2777                         }
2778
2779                         length -= read;
2780                 }
2781
2782                 EVBUFFER_LOCK(outbuf);
2783                 if (outbuf->freeze_end) {
2784                         evbuffer_free(tmp);
2785                         ok = 0;
2786                 } else {
2787                         evbuffer_add_buffer(outbuf, tmp);
2788                         evbuffer_free(tmp);
2789
2790 #ifdef WIN32
2791 #define close _close
2792 #endif
2793                         close(fd);
2794                 }
2795         }
2796
2797         if (ok)
2798                 evbuffer_invoke_callbacks(outbuf);
2799         EVBUFFER_UNLOCK(outbuf);
2800
2801         return ok ? 0 : -1;
2802 }
2803
2804
2805 void
2806 evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg)
2807 {
2808         EVBUFFER_LOCK(buffer);
2809
2810         if (!TAILQ_EMPTY(&buffer->callbacks))
2811                 evbuffer_remove_all_callbacks(buffer);
2812
2813         if (cb) {
2814                 struct evbuffer_cb_entry *ent =
2815                     evbuffer_add_cb(buffer, NULL, cbarg);
2816                 ent->cb.cb_obsolete = cb;
2817                 ent->flags |= EVBUFFER_CB_OBSOLETE;
2818         }
2819         EVBUFFER_UNLOCK(buffer);
2820 }
2821
2822 struct evbuffer_cb_entry *
2823 evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
2824 {
2825         struct evbuffer_cb_entry *e;
2826         if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry))))
2827                 return NULL;
2828         EVBUFFER_LOCK(buffer);
2829         e->cb.cb_func = cb;
2830         e->cbarg = cbarg;
2831         e->flags = EVBUFFER_CB_ENABLED;
2832         TAILQ_INSERT_HEAD(&buffer->callbacks, e, next);
2833         EVBUFFER_UNLOCK(buffer);
2834         return e;
2835 }
2836
2837 int
2838 evbuffer_remove_cb_entry(struct evbuffer *buffer,
2839                          struct evbuffer_cb_entry *ent)
2840 {
2841         EVBUFFER_LOCK(buffer);
2842         TAILQ_REMOVE(&buffer->callbacks, ent, next);
2843         EVBUFFER_UNLOCK(buffer);
2844         mm_free(ent);
2845         return 0;
2846 }
2847
2848 int
2849 evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
2850 {
2851         struct evbuffer_cb_entry *cbent;
2852         int result = -1;
2853         EVBUFFER_LOCK(buffer);
2854         TAILQ_FOREACH(cbent, &buffer->callbacks, next) {
2855                 if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) {
2856                         result = evbuffer_remove_cb_entry(buffer, cbent);
2857                         goto done;
2858                 }
2859         }
2860 done:
2861         EVBUFFER_UNLOCK(buffer);
2862         return result;
2863 }
2864
2865 int
2866 evbuffer_cb_set_flags(struct evbuffer *buffer,
2867                       struct evbuffer_cb_entry *cb, ev_uint32_t flags)
2868 {
2869         /* the user isn't allowed to mess with these. */
2870         flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
2871         EVBUFFER_LOCK(buffer);
2872         cb->flags |= flags;
2873         EVBUFFER_UNLOCK(buffer);
2874         return 0;
2875 }
2876
2877 int
2878 evbuffer_cb_clear_flags(struct evbuffer *buffer,
2879                       struct evbuffer_cb_entry *cb, ev_uint32_t flags)
2880 {
2881         /* the user isn't allowed to mess with these. */
2882         flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
2883         EVBUFFER_LOCK(buffer);
2884         cb->flags &= ~flags;
2885         EVBUFFER_UNLOCK(buffer);
2886         return 0;
2887 }
2888
2889 int
2890 evbuffer_freeze(struct evbuffer *buffer, int start)
2891 {
2892         EVBUFFER_LOCK(buffer);
2893         if (start)
2894                 buffer->freeze_start = 1;
2895         else
2896                 buffer->freeze_end = 1;
2897         EVBUFFER_UNLOCK(buffer);
2898         return 0;
2899 }
2900
2901 int
2902 evbuffer_unfreeze(struct evbuffer *buffer, int start)
2903 {
2904         EVBUFFER_LOCK(buffer);
2905         if (start)
2906                 buffer->freeze_start = 0;
2907         else
2908                 buffer->freeze_end = 0;
2909         EVBUFFER_UNLOCK(buffer);
2910         return 0;
2911 }
2912
2913 #if 0
2914 void
2915 evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
2916 {
2917         if (!(cb->flags & EVBUFFER_CB_SUSPENDED)) {
2918                 cb->size_before_suspend = evbuffer_get_length(buffer);
2919                 cb->flags |= EVBUFFER_CB_SUSPENDED;
2920         }
2921 }
2922
2923 void
2924 evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
2925 {
2926         if ((cb->flags & EVBUFFER_CB_SUSPENDED)) {
2927                 unsigned call = (cb->flags & EVBUFFER_CB_CALL_ON_UNSUSPEND);
2928                 size_t sz = cb->size_before_suspend;
2929                 cb->flags &= ~(EVBUFFER_CB_SUSPENDED|
2930                                EVBUFFER_CB_CALL_ON_UNSUSPEND);
2931                 cb->size_before_suspend = 0;
2932                 if (call && (cb->flags & EVBUFFER_CB_ENABLED)) {
2933                         cb->cb(buffer, sz, evbuffer_get_length(buffer), cb->cbarg);
2934                 }
2935         }
2936 }
2937 #endif
2938
2939 /* These hooks are exposed so that the unit tests can temporarily disable
2940  * sendfile support in order to test mmap, or both to test linear
2941  * access. Don't use it; if we need to add a way to disable sendfile support
2942  * in the future, it will probably be via an alternate version of
2943  * evbuffer_add_file() with a 'flags' argument.
2944  */
2945 int _evbuffer_testing_use_sendfile(void);
2946 int _evbuffer_testing_use_mmap(void);
2947 int _evbuffer_testing_use_linear_file_access(void);
2948
2949 int
2950 _evbuffer_testing_use_sendfile(void)
2951 {
2952         int ok = 0;
2953 #ifdef USE_SENDFILE
2954         use_sendfile = 1;
2955         ok = 1;
2956 #endif
2957 #ifdef _EVENT_HAVE_MMAP
2958         use_mmap = 0;
2959 #endif
2960         return ok;
2961 }
2962 int
2963 _evbuffer_testing_use_mmap(void)
2964 {
2965         int ok = 0;
2966 #ifdef USE_SENDFILE
2967         use_sendfile = 0;
2968 #endif
2969 #ifdef _EVENT_HAVE_MMAP
2970         use_mmap = 1;
2971         ok = 1;
2972 #endif
2973         return ok;
2974 }
2975 int
2976 _evbuffer_testing_use_linear_file_access(void)
2977 {
2978 #ifdef USE_SENDFILE
2979         use_sendfile = 0;
2980 #endif
2981 #ifdef _EVENT_HAVE_MMAP
2982         use_mmap = 0;
2983 #endif
2984         return 1;
2985 }