bfq-iosched.c

/*
 * Budget Fair Queueing (BFQ) disk scheduler.
 *
 * Based on ideas and code from CFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
 *
 * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ
 * file.
 *
 * BFQ is a proportional-share storage-I/O scheduling algorithm based on
 * the slice-by-slice service scheme of CFQ. But BFQ assigns budgets,
 * measured in number of sectors, to processes instead of time slices. The
 * device is not granted to the in-service process for a given time slice,
 * but until it has exhausted its assigned budget. This change from the time
 * to the service domain allows BFQ to distribute the device throughput
 * among processes as desired, without any distortion due to ZBR, workload
 * fluctuations or other factors. BFQ uses an ad hoc internal scheduler,
 * called B-WF2Q+, to schedule processes according to their budgets. More
 * precisely, BFQ schedules queues associated to processes. Thanks to the
 * accurate policy of B-WF2Q+, BFQ can afford to assign high budgets to
 * I/O-bound processes issuing sequential requests (to boost the
 * throughput), and yet guarantee a low latency to interactive and soft
 * real-time applications.
 *
 * BFQ is described in [1], where also a reference to the initial, more
 * theoretical paper on BFQ can be found. The interested reader can find
 * in the latter paper full details on the main algorithm, as well as
 * formulas of the guarantees and formal proofs of all the properties.
 * With respect to the version of BFQ presented in these papers, this
 * implementation adds a few more heuristics, such as the one that
 * guarantees a low latency to soft real-time applications, and a
 * hierarchical extension based on H-WF2Q+.
 *
 * B-WF2Q+ is based on WF2Q+, that is described in [2], together with
 * H-WF2Q+, while the augmented tree used to implement B-WF2Q+ with O(log N)
 * complexity derives from the one introduced with EEVDF in [3].
 *
 * [1] P. Valente and M. Andreolini, ``Improving Application Responsiveness
 *     with the BFQ Disk I/O Scheduler'',
 *     Proceedings of the 5th Annual International Systems and Storage
 *     Conference (SYSTOR '12), June 2012.
 *
 * http://algogroup.unimo.it/people/paolo/disk_sched/bf1-v1-suite-results.pdf
 *
 * [2] Jon C.R. Bennett and H. Zhang, ``Hierarchical Packet Fair Queueing
 *     Algorithms,'' IEEE/ACM Transactions on Networking, 5(5):675-689,
 *     Oct 1997.
 *
 * http://www.cs.cmu.edu/~hzhang/papers/TON-97-Oct.ps.gz
 *
 * [3] I. Stoica and H. Abdel-Wahab, ``Earliest Eligible Virtual Deadline
 *     First: A Flexible and Accurate Mechanism for Proportional Share
 *     Resource Allocation,'' technical report.
 *
 * http://www.cs.berkeley.edu/~istoica/papers/eevdf-tr-95.pdf
 */
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/cgroup.h>
#include <linux/elevator.h>
#include <linux/jiffies.h>
#include <linux/rbtree.h>
#include <linux/ioprio.h>
#include "bfq.h"
#include "blk.h"

/* Expiration time of sync (0) and async (1) requests, in jiffies. */
static const int bfq_fifo_expire[2] = { HZ / 4, HZ / 8 };

/* Maximum backwards seek, in KiB. */
static const int bfq_back_max = 16 * 1024;

/* Penalty of a backwards seek, in number of sectors. */
static const int bfq_back_penalty = 2;

/* Idling period duration, in jiffies. */
static int bfq_slice_idle = HZ / 125;

/* Default maximum budget values, in sectors and number of requests. */
static const int bfq_default_max_budget = 16 * 1024;
static const int bfq_max_budget_async_rq = 4;

/*
 * Async to sync throughput distribution is controlled as follows:
 * when an async request is served, the entity is charged the number
 * of sectors of the request, multiplied by the factor below
 */
static const int bfq_async_charge_factor = 10;

/* Default timeout values, in jiffies, approximating CFQ defaults. */
static const int bfq_timeout_sync = HZ / 8;
static int bfq_timeout_async = HZ / 25;

struct kmem_cache *bfq_pool;

/* Below this threshold (in ms), we consider thinktime immediate. */
#define BFQ_MIN_TT		2

/* hw_tag detection: parallel requests threshold and min samples needed. */
#define BFQ_HW_QUEUE_THRESHOLD	4
#define BFQ_HW_QUEUE_SAMPLES	32

#define BFQQ_SEEK_THR	 (sector_t)(8 * 1024)
#define BFQQ_SEEKY(bfqq) ((bfqq)->seek_mean > BFQQ_SEEK_THR)

/* Min samples used for peak rate estimation (for autotuning). */
#define BFQ_PEAK_RATE_SAMPLES	32

/* Shift used for peak rate fixed precision calculations. */
#define BFQ_RATE_SHIFT		16

/*
 * By default, BFQ computes the duration of the weight raising for
 * interactive applications automatically, using the following formula:
 * duration = (R / r) * T, where r is the peak rate of the device, and
 * R and T are two reference parameters.
 * In particular, R is the peak rate of the reference device (see below),
 * and T is a reference time: given the systems that are likely to be
 * installed on the reference device according to its speed class, T is
 * about the maximum time needed, under BFQ and while reading two files in
 * parallel, to load typical large applications on these systems.
 * In practice, the slower/faster the device at hand is, the more/less it
 * takes to load applications with respect to the reference device.
 * Accordingly, the longer/shorter BFQ grants weight raising to interactive
 * applications.
 *
 * BFQ uses four different reference pairs (R, T), depending on:
 * . whether the device is rotational or non-rotational;
 * . whether the device is slow, such as old or portable HDDs, as well as
 *   SD cards, or fast, such as newer HDDs and SSDs.
 *
 * The device's speed class is dynamically (re)detected in
 * bfq_update_peak_rate() every time the estimated peak rate is updated.
 *
 * In the following definitions, R_slow[0]/R_fast[0] and T_slow[0]/T_fast[0]
 * are the reference values for a slow/fast rotational device, whereas
 * R_slow[1]/R_fast[1] and T_slow[1]/T_fast[1] are the reference values for
 * a slow/fast non-rotational device. Finally, device_speed_thresh are the
 * thresholds used to switch between speed classes.
 * Both the reference peak rates and the thresholds are measured in
 * sectors/usec, left-shifted by BFQ_RATE_SHIFT.
 */
static int R_slow[2] = {1536, 10752};
static int R_fast[2] = {17415, 34791};
/*
 * To improve readability, a conversion function is used to initialize the
 * following arrays, which entails that they can be initialized only in a
 * function.
 */
static int T_slow[2];
static int T_fast[2];
static int device_speed_thresh[2];

#define BFQ_SERVICE_TREE_INIT	((struct bfq_service_tree)		\
				{ RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 })

#define RQ_BIC(rq)		((struct bfq_io_cq *) (rq)->elv.priv[0])
#define RQ_BFQQ(rq)		((rq)->elv.priv[1])

static inline void bfq_schedule_dispatch(struct bfq_data *bfqd);

#include "bfq-ioc.c"
#include "bfq-sched.c"
#include "bfq-cgroup.c"

#define bfq_class_idle(bfqq)	((bfqq)->entity.ioprio_class ==\
				 IOPRIO_CLASS_IDLE)
#define bfq_class_rt(bfqq)	((bfqq)->entity.ioprio_class ==\
				 IOPRIO_CLASS_RT)

#define bfq_sample_valid(samples)	((samples) > 80)

/*
 * The following macro groups conditions that need to be evaluated when
 * checking if existing queues and groups form a symmetric scenario
 * and therefore idling can be reduced or disabled for some of the
 * queues. See the comment to the function bfq_bfqq_must_not_expire()
 * for further details.
 */
#ifdef CONFIG_CGROUP_BFQIO
#define symmetric_scenario	  (!bfqd->active_numerous_groups && \
				   !bfq_differentiated_weights(bfqd))
#else
#define symmetric_scenario	  (!bfq_differentiated_weights(bfqd))
#endif

/*
 * We regard a request as SYNC, if either it's a read or has the SYNC bit
 * set (in which case it could also be a direct WRITE).
 */
static inline int bfq_bio_sync(struct bio *bio)
{
	if (bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC))
		return 1;

	return 0;
}

/*
 * Scheduler run of queue, if there are requests pending and no one in the
 * driver that will restart queueing.
 */
static inline void bfq_schedule_dispatch(struct bfq_data *bfqd)
{
	if (bfqd->queued != 0) {
		bfq_log(bfqd, "schedule dispatch");
		kblockd_schedule_work(&bfqd->unplug_work);
	}
}

/*
 * Lifted from AS - choose which of rq1 and rq2 that is best served now.
 * We choose the request that is closesr to the head right now.  Distance
 * behind the head is penalized and only allowed to a certain extent.
 */
static struct request *bfq_choose_req(struct bfq_data *bfqd,
				      struct request *rq1,
				      struct request *rq2,
				      sector_t last)
{
	sector_t s1, s2, d1 = 0, d2 = 0;
	unsigned long back_max;
#define BFQ_RQ1_WRAP	0x01 /* request 1 wraps */
#define BFQ_RQ2_WRAP	0x02 /* request 2 wraps */
	unsigned wrap = 0; /* bit mask: requests behind the disk head? */

	if (rq1 == NULL || rq1 == rq2)
		return rq2;
	if (rq2 == NULL)
		return rq1;

	if (rq_is_sync(rq1) && !rq_is_sync(rq2))
		return rq1;
	else if (rq_is_sync(rq2) && !rq_is_sync(rq1))
		return rq2;
	if ((rq1->cmd_flags & REQ_META) && !(rq2->cmd_flags & REQ_META))
		return rq1;
	else if ((rq2->cmd_flags & REQ_META) && !(rq1->cmd_flags & REQ_META))
		return rq2;

	s1 = blk_rq_pos(rq1);
	s2 = blk_rq_pos(rq2);

	/*
	 * By definition, 1KiB is 2 sectors.
	 */
	back_max = bfqd->bfq_back_max * 2;

	/*
	 * Strict one way elevator _except_ in the case where we allow
	 * short backward seeks which are biased as twice the cost of a
	 * similar forward seek.
	 */
	if (s1 >= last)
		d1 = s1 - last;
	else if (s1 + back_max >= last)
		d1 = (last - s1) * bfqd->bfq_back_penalty;
	else
		wrap |= BFQ_RQ1_WRAP;

	if (s2 >= last)
		d2 = s2 - last;
	else if (s2 + back_max >= last)
		d2 = (last - s2) * bfqd->bfq_back_penalty;
	else
		wrap |= BFQ_RQ2_WRAP;

	/* Found required data */

	/*
	 * By doing switch() on the bit mask "wrap" we avoid having to
	 * check two variables for all permutations: --> faster!
	 */
	switch (wrap) {
	case 0: /* common case for CFQ: rq1 and rq2 not wrapped */
		if (d1 < d2)
			return rq1;
		else if (d2 < d1)
			return rq2;
		else {
			if (s1 >= s2)
				return rq1;
			else
				return rq2;
		}

	case BFQ_RQ2_WRAP:
		return rq1;
	case BFQ_RQ1_WRAP:
		return rq2;
	case (BFQ_RQ1_WRAP|BFQ_RQ2_WRAP): /* both rqs wrapped */
	default:
		/*
		 * Since both rqs are wrapped,
		 * start with the one that's further behind head
		 * (--> only *one* back seek required),
		 * since back seek takes more time than forward.
		 */
		if (s1 <= s2)
			return rq1;
		else
			return rq2;
	}
}

static struct bfq_queue *
bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root,
		     sector_t sector, struct rb_node **ret_parent,
		     struct rb_node ***rb_link)
{
	struct rb_node **p, *parent;
	struct bfq_queue *bfqq = NULL;

	parent = NULL;
	p = &root->rb_node;
	while (*p) {
		struct rb_node **n;

		parent = *p;
		bfqq = rb_entry(parent, struct bfq_queue, pos_node);

		/*
		 * Sort strictly based on sector. Smallest to the left,
		 * largest to the right.
		 */
		if (sector > blk_rq_pos(bfqq->next_rq))
			n = &(*p)->rb_right;
		else if (sector < blk_rq_pos(bfqq->next_rq))
			n = &(*p)->rb_left;
		else
			break;
		p = n;
		bfqq = NULL;
	}

	*ret_parent = parent;
	if (rb_link)
		*rb_link = p;

	bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d",
		(long long unsigned)sector,
		bfqq != NULL ? bfqq->pid : 0);

	return bfqq;
}

static void bfq_rq_pos_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq)
{
	struct rb_node **p, *parent;
	struct bfq_queue *__bfqq;

	if (bfqq->pos_root != NULL) {
		rb_erase(&bfqq->pos_node, bfqq->pos_root);
		bfqq->pos_root = NULL;
	}

	if (bfq_class_idle(bfqq))
		return;
	if (!bfqq->next_rq)
		return;

	bfqq->pos_root = &bfqd->rq_pos_tree;
	__bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root,
			blk_rq_pos(bfqq->next_rq), &parent, &p);
	if (__bfqq == NULL) {
		rb_link_node(&bfqq->pos_node, parent, p);
		rb_insert_color(&bfqq->pos_node, bfqq->pos_root);
	} else
		bfqq->pos_root = NULL;
}

/*
 * Tell whether there are active queues or groups with differentiated weights.
 */
static inline bool bfq_differentiated_weights(struct bfq_data *bfqd)
{
	/*
	 * For weights to differ, at least one of the trees must contain
	 * at least two nodes.
	 */
	return (!RB_EMPTY_ROOT(&bfqd->queue_weights_tree) &&
		(bfqd->queue_weights_tree.rb_node->rb_left ||
		 bfqd->queue_weights_tree.rb_node->rb_right)
#ifdef CONFIG_CGROUP_BFQIO
	       ) ||
	       (!RB_EMPTY_ROOT(&bfqd->group_weights_tree) &&
		(bfqd->group_weights_tree.rb_node->rb_left ||
		 bfqd->group_weights_tree.rb_node->rb_right)
#endif
	       );
}

/*
 * If the weight-counter tree passed as input contains no counter for
 * the weight of the input entity, then add that counter; otherwise just
 * increment the existing counter.
 *
 * Note that weight-counter trees contain few nodes in mostly symmetric
 * scenarios. For example, if all queues have the same weight, then the
 * weight-counter tree for the queues may contain at most one node.
 * This holds even if low_latency is on, because weight-raised queues
 * are not inserted in the tree.
 * In most scenarios, the rate at which nodes are created/destroyed
 * should be low too.
 */
static void bfq_weights_tree_add(struct bfq_data *bfqd,
				 struct bfq_entity *entity,
				 struct rb_root *root)
{
	struct rb_node **new = &(root->rb_node), *parent = NULL;

	/*
	 * Do not insert if the entity is already associated with a
	 * counter, which happens if:
	 *   1) the entity is associated with a queue,
	 *   2) a request arrival has caused the queue to become both
	 *      non-weight-raised, and hence change its weight, and
	 *      backlogged; in this respect, each of the two events
	 *      causes an invocation of this function,
	 *   3) this is the invocation of this function caused by the
	 *      second event. This second invocation is actually useless,
	 *      and we handle this fact by exiting immediately. More
	 *      efficient or clearer solutions might possibly be adopted.
	 */
	if (entity->weight_counter)
		return;

	while (*new) {
		struct bfq_weight_counter *__counter = container_of(*new,
						struct bfq_weight_counter,
						weights_node);
		parent = *new;

		if (entity->weight == __counter->weight) {
			entity->weight_counter = __counter;
			goto inc_counter;
		}
		if (entity->weight < __counter->weight)
			new = &((*new)->rb_left);
		else
			new = &((*new)->rb_right);
	}

	entity->weight_counter = kzalloc(sizeof(struct bfq_weight_counter),
					 GFP_ATOMIC);
	entity->weight_counter->weight = entity->weight;
	rb_link_node(&entity->weight_counter->weights_node, parent, new);
	rb_insert_color(&entity->weight_counter->weights_node, root);

inc_counter:
	entity->weight_counter->num_active++;
}

/*
 * Decrement the weight counter associated with the entity, and, if the
 * counter reaches 0, remove the counter from the tree.
 * See the comments to the function bfq_weights_tree_add() for considerations
 * about overhead.
 */
static void bfq_weights_tree_remove(struct bfq_data *bfqd,
				    struct bfq_entity *entity,
				    struct rb_root *root)
{
	if (!entity->weight_counter)
		return;

	BUG_ON(RB_EMPTY_ROOT(root));
	BUG_ON(entity->weight_counter->weight != entity->weight);

	BUG_ON(!entity->weight_counter->num_active);
	entity->weight_counter->num_active--;
	if (entity->weight_counter->num_active > 0)
		goto reset_entity_pointer;

	rb_erase(&entity->weight_counter->weights_node, root);
	kfree(entity->weight_counter);

reset_entity_pointer:
	entity->weight_counter = NULL;
}

static struct request *bfq_find_next_rq(struct bfq_data *bfqd,
					struct bfq_queue *bfqq,
					struct request *last)
{
	struct rb_node *rbnext = rb_next(&last->rb_node);
	struct rb_node *rbprev = rb_prev(&last->rb_node);
	struct request *next = NULL, *prev = NULL;

	BUG_ON(RB_EMPTY_NODE(&last->rb_node));

	if (rbprev != NULL)
		prev = rb_entry_rq(rbprev);

	if (rbnext != NULL)
		next = rb_entry_rq(rbnext);
	else {
		rbnext = rb_first(&bfqq->sort_list);
		if (rbnext && rbnext != &last->rb_node)
			next = rb_entry_rq(rbnext);
	}

	return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last));
}

/* see the definition of bfq_async_charge_factor for details */
static inline unsigned long bfq_serv_to_charge(struct request *rq,
					       struct bfq_queue *bfqq)
{
	return blk_rq_sectors(rq) *
		(1 + ((!bfq_bfqq_sync(bfqq)) * (bfqq->wr_coeff == 1) *
		bfq_async_charge_factor));
}

/**
 * bfq_updated_next_req - update the queue after a new next_rq selection.
 * @bfqd: the device data the queue belongs to.
 * @bfqq: the queue to update.
 *
 * If the first request of a queue changes we make sure that the queue
 * has enough budget to serve at least its first request (if the
 * request has grown).  We do this because if the queue has not enough
 * budget for its first request, it has to go through two dispatch
 * rounds to actually get it dispatched.
 */
static void bfq_updated_next_req(struct bfq_data *bfqd,
				 struct bfq_queue *bfqq)
{
	struct bfq_entity *entity = &bfqq->entity;
	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
	struct request *next_rq = bfqq->next_rq;
	unsigned long new_budget;

	if (next_rq == NULL)
		return;

	if (bfqq == bfqd->in_service_queue)
		/*
		 * In order not to break guarantees, budgets cannot be
		 * changed after an entity has been selected.
		 */
		return;

	BUG_ON(entity->tree != &st->active);
	BUG_ON(entity == entity->sched_data->in_service_entity);

	new_budget = max_t(unsigned long, bfqq->max_budget,
			   bfq_serv_to_charge(next_rq, bfqq));
	if (entity->budget != new_budget) {
		entity->budget = new_budget;
		bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu",
					 new_budget);
		bfq_activate_bfqq(bfqd, bfqq);
	}
}

static inline unsigned int bfq_wr_duration(struct bfq_data *bfqd)
{
	u64 dur;

	if (bfqd->bfq_wr_max_time > 0)
		return bfqd->bfq_wr_max_time;

	dur = bfqd->RT_prod;
	do_div(dur, bfqd->peak_rate);

	return dur;
}

static inline unsigned
bfq_bfqq_cooperations(struct bfq_queue *bfqq)
{
	return bfqq->bic ? bfqq->bic->cooperations : 0;
}

static inline void
bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
{
	if (bic->saved_idle_window)
		bfq_mark_bfqq_idle_window(bfqq);
	else
		bfq_clear_bfqq_idle_window(bfqq);
	if (bic->saved_IO_bound)
		bfq_mark_bfqq_IO_bound(bfqq);
	else
		bfq_clear_bfqq_IO_bound(bfqq);
	/* Assuming that the flag in_large_burst is already correctly set */
	if (bic->wr_time_left && bfqq->bfqd->low_latency &&
	    !bfq_bfqq_in_large_burst(bfqq) &&
	    bic->cooperations < bfqq->bfqd->bfq_coop_thresh) {
		/*
		 * Start a weight raising period with the duration given by
		 * the raising_time_left snapshot.
		 */
		if (bfq_bfqq_busy(bfqq))
			bfqq->bfqd->wr_busy_queues++;
		bfqq->wr_coeff = bfqq->bfqd->bfq_wr_coeff;
		bfqq->wr_cur_max_time = bic->wr_time_left;
		bfqq->last_wr_start_finish = jiffies;
		bfqq->entity.ioprio_changed = 1;
	}
	/*
	 * Clear wr_time_left to prevent bfq_bfqq_save_state() from
	 * getting confused about the queue's need of a weight-raising
	 * period.
	 */
	bic->wr_time_left = 0;
}

/* Must be called with the queue_lock held. */
static int bfqq_process_refs(struct bfq_queue *bfqq)
{
	int process_refs, io_refs;

	io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE];
	process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st;
	BUG_ON(process_refs < 0);
	return process_refs;
}

/* Empty burst list and add just bfqq (see comments to bfq_handle_burst) */
static inline void bfq_reset_burst_list(struct bfq_data *bfqd,
					struct bfq_queue *bfqq)
{
	struct bfq_queue *item;
	struct hlist_node *n;

	hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node)
		hlist_del_init(&item->burst_list_node);
	hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);
	bfqd->burst_size = 1;
}

/* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */
static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
{
	/* Increment burst size to take into account also bfqq */
	bfqd->burst_size++;

	if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) {
		struct bfq_queue *pos, *bfqq_item;
		struct hlist_node *n;

		/*
		 * Enough queues have been activated shortly after each
		 * other to consider this burst as large.
		 */
		bfqd->large_burst = true;

		/*
		 * We can now mark all queues in the burst list as
		 * belonging to a large burst.
		 */
		hlist_for_each_entry(bfqq_item, &bfqd->burst_list,
				     burst_list_node)
		        bfq_mark_bfqq_in_large_burst(bfqq_item);
		bfq_mark_bfqq_in_large_burst(bfqq);

		/*
		 * From now on, and until the current burst finishes, any
		 * new queue being activated shortly after the last queue
		 * was inserted in the burst can be immediately marked as
		 * belonging to a large burst. So the burst list is not
		 * needed any more. Remove it.
		 */
		hlist_for_each_entry_safe(pos, n, &bfqd->burst_list,
					  burst_list_node)
			hlist_del_init(&pos->burst_list_node);
	} else /* burst not yet large: add bfqq to the burst list */
		hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);
}

/*
 * If many queues happen to become active shortly after each other, then,
 * to help the processes associated to these queues get their job done as
 * soon as possible, it is usually better to not grant either weight-raising
 * or device idling to these queues. In this comment we describe, firstly,
 * the reasons why this fact holds, and, secondly, the next function, which
 * implements the main steps needed to properly mark these queues so that
 * they can then be treated in a different way.
 *
 * As for the terminology, we say that a queue becomes active, i.e.,
 * switches from idle to backlogged, either when it is created (as a
 * consequence of the arrival of an I/O request), or, if already existing,
 * when a new request for the queue arrives while the queue is idle.
 * Bursts of activations, i.e., activations of different queues occurring
 * shortly after each other, are typically caused by services or applications
 * that spawn or reactivate many parallel threads/processes. Examples are
 * systemd during boot or git grep.
 *
 * These services or applications benefit mostly from a high throughput:
 * the quicker the requests of the activated queues are cumulatively served,
 * the sooner the target job of these queues gets completed. As a consequence,
 * weight-raising any of these queues, which also implies idling the device
 * for it, is almost always counterproductive: in most cases it just lowers
 * throughput.
 *
 * On the other hand, a burst of activations may be also caused by the start
 * of an application that does not consist in a lot of parallel I/O-bound
 * threads. In fact, with a complex application, the burst may be just a
 * consequence of the fact that several processes need to be executed to
 * start-up the application. To start an application as quickly as possible,
 * the best thing to do is to privilege the I/O related to the application
 * with respect to all other I/O. Therefore, the best strategy to start as
 * quickly as possible an application that causes a burst of activations is
 * to weight-raise all the queues activated during the burst. This is the
 * exact opposite of the best strategy for the other type of bursts.
 *
 * In the end, to take the best action for each of the two cases, the two
 * types of bursts need to be distinguished. Fortunately, this seems
 * relatively easy to do, by looking at the sizes of the bursts. In
 * particular, we found a threshold such that bursts with a larger size
 * than that threshold are apparently caused only by services or commands
 * such as systemd or git grep. For brevity, hereafter we call just 'large'
 * these bursts. BFQ *does not* weight-raise queues whose activations occur
 * in a large burst. In addition, for each of these queues BFQ performs or
 * does not perform idling depending on which choice boosts the throughput
 * most. The exact choice depends on the device and request pattern at
 * hand.
 *
 * Turning back to the next function, it implements all the steps needed
 * to detect the occurrence of a large burst and to properly mark all the
 * queues belonging to it (so that they can then be treated in a different
 * way). This goal is achieved by maintaining a special "burst list" that
 * holds, temporarily, the queues that belong to the burst in progress. The
 * list is then used to mark these queues as belonging to a large burst if
 * the burst does become large. The main steps are the following.
 *
 * . when the very first queue is activated, the queue is inserted into the
 *   list (as it could be the first queue in a possible burst)
 *
 * . if the current burst has not yet become large, and a queue Q that does
 *   not yet belong to the burst is activated shortly after the last time
 *   at which a new queue entered the burst list, then the function appends
 *   Q to the burst list
 *
 * . if, as a consequence of the previous step, the burst size reaches
 *   the large-burst threshold, then
 *
 *     . all the queues in the burst list are marked as belonging to a
 *       large burst
 *
 *     . the burst list is deleted; in fact, the burst list already served
 *       its purpose (keeping temporarily track of the queues in a burst,
 *       so as to be able to mark them as belonging to a large burst in the
 *       previous sub-step), and now is not needed any more
 *
 *     . the device enters a large-burst mode
 *
 * . if a queue Q that does not belong to the burst is activated while
 *   the device is in large-burst mode and shortly after the last time
 *   at which a queue either entered the burst list or was marked as
 *   belonging to the current large burst, then Q is immediately marked
 *   as belonging to a large burst.
 *
 * . if a queue Q that does not belong to the burst is activated a while
 *   later, i.e., not shortly after, than the last time at which a queue
 *   either entered the burst list or was marked as belonging to the
 *   current large burst, then the current burst is deemed as finished and:
 *
 *        . the large-burst mode is reset if set
 *
 *        . the burst list is emptied
 *
 *        . Q is inserted in the burst list, as Q may be the first queue
 *          in a possible new burst (then the burst list contains just Q
 *          after this step).
 */
static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
			     bool idle_for_long_time)
{
	/*
	 * If bfqq happened to be activated in a burst, but has been idle
	 * for at least as long as an interactive queue, then we assume
	 * that, in the overall I/O initiated in the burst, the I/O
	 * associated to bfqq is finished. So bfqq does not need to be
	 * treated as a queue belonging to a burst anymore. Accordingly,
	 * we reset bfqq's in_large_burst flag if set, and remove bfqq
	 * from the burst list if it's there. We do not decrement instead
	 * burst_size, because the fact that bfqq does not need to belong
	 * to the burst list any more does not invalidate the fact that
	 * bfqq may have been activated during the current burst.
	 */
	if (idle_for_long_time) {
		hlist_del_init(&bfqq->burst_list_node);
		bfq_clear_bfqq_in_large_burst(bfqq);
	}

	/*
	 * If bfqq is already in the burst list or is part of a large
	 * burst, then there is nothing else to do.
	 */
	if (!hlist_unhashed(&bfqq->burst_list_node) ||
	    bfq_bfqq_in_large_burst(bfqq))
		return;

	/*
	 * If bfqq's activation happens late enough, then the current
	 * burst is finished, and related data structures must be reset.
	 *
	 * In this respect, consider the special case where bfqq is the very
	 * first queue being activated. In this case, last_ins_in_burst is
	 * not yet significant when we get here. But it is easy to verify
	 * that, whether or not the following condition is true, bfqq will
	 * end up being inserted into the burst list. In particular the
	 * list will happen to contain only bfqq. And this is exactly what
	 * has to happen, as bfqq may be the first queue in a possible
	 * burst.
	 */
	if (time_is_before_jiffies(bfqd->last_ins_in_burst +
	    bfqd->bfq_burst_interval)) {
		bfqd->large_burst = false;
		bfq_reset_burst_list(bfqd, bfqq);
		return;
	}

	/*
	 * If we get here, then bfqq is being activated shortly after the
	 * last queue. So, if the current burst is also large, we can mark
	 * bfqq as belonging to this large burst immediately.
	 */
	if (bfqd->large_burst) {
		bfq_mark_bfqq_in_large_burst(bfqq);
		return;
	}

	/*
	 * If we get here, then a large-burst state has not yet been
	 * reached, but bfqq is being activated shortly after the last
	 * queue. Then we add bfqq to the burst.
	 */
	bfq_add_to_burst(bfqd, bfqq);
}

static void bfq_add_request(struct request *rq)
{
	struct bfq_queue *bfqq = RQ_BFQQ(rq);
	struct bfq_entity *entity = &bfqq->entity;
	struct bfq_data *bfqd = bfqq->bfqd;
	struct request *next_rq, *prev;
	unsigned long old_wr_coeff = bfqq->wr_coeff;
	bool interactive = false;

	bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));
	bfqq->queued[rq_is_sync(rq)]++;
	bfqd->queued++;

	elv_rb_add(&bfqq->sort_list, rq);

	/*
	 * Check if this request is a better next-serve candidate.
	 */
	prev = bfqq->next_rq;
	next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position);
	BUG_ON(next_rq == NULL);
	bfqq->next_rq = next_rq;

	/*
	 * Adjust priority tree position, if next_rq changes.
	 */
	if (prev != bfqq->next_rq)
		bfq_rq_pos_tree_add(bfqd, bfqq);

	if (!bfq_bfqq_busy(bfqq)) {
		bool soft_rt, coop_or_in_burst,
		     idle_for_long_time = time_is_before_jiffies(
						bfqq->budget_timeout +
						bfqd->bfq_wr_min_idle_time);

		if (bfq_bfqq_sync(bfqq)) {
			bool already_in_burst =
			   !hlist_unhashed(&bfqq->burst_list_node) ||
			   bfq_bfqq_in_large_burst(bfqq);
			bfq_handle_burst(bfqd, bfqq, idle_for_long_time);
			/*
			 * If bfqq was not already in the current burst,
			 * then, at this point, bfqq either has been
			 * added to the current burst or has caused the
			 * current burst to terminate. In particular, in
			 * the second case, bfqq has become the first
			 * queue in a possible new burst.
			 * In both cases last_ins_in_burst needs to be
			 * moved forward.
			 */
			if (!already_in_burst)
				bfqd->last_ins_in_burst = jiffies;
		}

		coop_or_in_burst = bfq_bfqq_in_large_burst(bfqq) ||
			bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh;
		soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 &&
			!coop_or_in_burst &&
			time_is_before_jiffies(bfqq->soft_rt_next_start);
		interactive = !coop_or_in_burst && idle_for_long_time;
		entity->budget = max_t(unsigned long, bfqq->max_budget,
				       bfq_serv_to_charge(next_rq, bfqq));

		if (!bfq_bfqq_IO_bound(bfqq)) {
			if (time_before(jiffies,
					RQ_BIC(rq)->ttime.last_end_request +
					bfqd->bfq_slice_idle)) {
				bfqq->requests_within_timer++;
				if (bfqq->requests_within_timer >=
				    bfqd->bfq_requests_within_timer)
					bfq_mark_bfqq_IO_bound(bfqq);
			} else
				bfqq->requests_within_timer = 0;
		}

		if (!bfqd->low_latency)
			goto add_bfqq_busy;

		if (bfq_bfqq_just_split(bfqq))
			goto set_ioprio_changed;

		/*
		 * If the queue:
		 * - is not being boosted,
		 * - has been idle for enough time,
		 * - is not a sync queue or is linked to a bfq_io_cq (it is
		 *   shared "for its nature" or it is not shared and its
		 *   requests have not been redirected to a shared queue)
		 * start a weight-raising period.
		 */
		if (old_wr_coeff == 1 && (interactive || soft_rt) &&
		    (!bfq_bfqq_sync(bfqq) || bfqq->bic != NULL)) {
			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
			if (interactive)
				bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
			else
				bfqq->wr_cur_max_time =
					bfqd->bfq_wr_rt_max_time;
			bfq_log_bfqq(bfqd, bfqq,
				     "wrais starting at %lu, rais_max_time %u",
				     jiffies,
				     jiffies_to_msecs(bfqq->wr_cur_max_time));
		} else if (old_wr_coeff > 1) {
			if (interactive)
				bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
			else if (coop_or_in_burst ||
				 (bfqq->wr_cur_max_time ==
				  bfqd->bfq_wr_rt_max_time &&
				  !soft_rt)) {
				bfqq->wr_coeff = 1;
				bfq_log_bfqq(bfqd, bfqq,
					"wrais ending at %lu, rais_max_time %u",
					jiffies,
					jiffies_to_msecs(bfqq->
						wr_cur_max_time));
			} else if (time_before(
					bfqq->last_wr_start_finish +
					bfqq->wr_cur_max_time,
					jiffies +
					bfqd->bfq_wr_rt_max_time) &&
				   soft_rt) {
				/*
				 *
				 * The remaining weight-raising time is lower
				 * than bfqd->bfq_wr_rt_max_time, which means
				 * that the application is enjoying weight
				 * raising either because deemed soft-rt in
				 * the near past, or because deemed interactive
				 * a long ago.
				 * In both cases, resetting now the current
				 * remaining weight-raising time for the
				 * application to the weight-raising duration
				 * for soft rt applications would not cause any
				 * latency increase for the application (as the
				 * new duration would be higher than the
				 * remaining time).
				 *
				 * In addition, the application is now meeting
				 * the requirements for being deemed soft rt.
				 * In the end we can correctly and safely
				 * (re)charge the weight-raising duration for
				 * the application with the weight-raising
				 * duration for soft rt applications.
				 *
				 * In particular, doing this recharge now, i.e.,
				 * before the weight-raising period for the
				 * application finishes, reduces the probability
				 * of the following negative scenario:
				 * 1) the weight of a soft rt application is
				 *    raised at startup (as for any newly
				 *    created application),
				 * 2) since the application is not interactive,
				 *    at a certain time weight-raising is
				 *    stopped for the application,
				 * 3) at that time the application happens to
				 *    still have pending requests, and hence
				 *    is destined to not have a chance to be
				 *    deemed soft rt before these requests are
				 *    completed (see the comments to the
				 *    function bfq_bfqq_softrt_next_start()
				 *    for details on soft rt detection),
				 * 4) these pending requests experience a high
				 *    latency because the application is not
				 *    weight-raised while they are pending.
				 */
				bfqq->last_wr_start_finish = jiffies;
				bfqq->wr_cur_max_time =
					bfqd->bfq_wr_rt_max_time;
			}
		}
set_ioprio_changed:
		if (old_wr_coeff != bfqq->wr_coeff)
			entity->ioprio_changed = 1;
add_bfqq_busy:
		bfqq->last_idle_bklogged = jiffies;
		bfqq->service_from_backlogged = 0;
		bfq_clear_bfqq_softrt_update(bfqq);
		bfq_add_bfqq_busy(bfqd, bfqq);
	} else {
		if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) &&
		    time_is_before_jiffies(
				bfqq->last_wr_start_finish +
				bfqd->bfq_wr_min_inter_arr_async)) {
			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);

			bfqd->wr_busy_queues++;
			entity->ioprio_changed = 1;
			bfq_log_bfqq(bfqd, bfqq,
			    "non-idle wrais starting at %lu, rais_max_time %u",
			    jiffies,
			    jiffies_to_msecs(bfqq->wr_cur_max_time));
		}
		if (prev != bfqq->next_rq)
			bfq_updated_next_req(bfqd, bfqq);
	}

	if (bfqd->low_latency &&
		(old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive))
		bfqq->last_wr_start_finish = jiffies;
}

static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd,
					  struct bio *bio)
{
	struct task_struct *tsk = current;
	struct bfq_io_cq *bic;
	struct bfq_queue *bfqq;

	bic = bfq_bic_lookup(bfqd, tsk->io_context);
	if (bic == NULL)
		return NULL;

	bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio));
	if (bfqq != NULL)
		return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio));

	return NULL;
}

static void bfq_activate_request(struct request_queue *q, struct request *rq)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;

	bfqd->rq_in_driver++;
	bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
	bfq_log(bfqd, "activate_request: new bfqd->last_position %llu",
		(long long unsigned)bfqd->last_position);
}

static inline void bfq_deactivate_request(struct request_queue *q,
					  struct request *rq)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;

	BUG_ON(bfqd->rq_in_driver == 0);
	bfqd->rq_in_driver--;
}

static void bfq_remove_request(struct request *rq)
{
	struct bfq_queue *bfqq = RQ_BFQQ(rq);
	struct bfq_data *bfqd = bfqq->bfqd;
	const int sync = rq_is_sync(rq);

	if (bfqq->next_rq == rq) {
		bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq);
		bfq_updated_next_req(bfqd, bfqq);
	}

	if (rq->queuelist.prev != &rq->queuelist)
		list_del_init(&rq->queuelist);
	BUG_ON(bfqq->queued[sync] == 0);
	bfqq->queued[sync]--;
	bfqd->queued--;
	elv_rb_del(&bfqq->sort_list, rq);

	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
		if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue)
			bfq_del_bfqq_busy(bfqd, bfqq, 1);
		/*
		 * Remove queue from request-position tree as it is empty.
		 */
		if (bfqq->pos_root != NULL) {
			rb_erase(&bfqq->pos_node, bfqq->pos_root);
			bfqq->pos_root = NULL;
		}
	}

	if (rq->cmd_flags & REQ_META) {
		BUG_ON(bfqq->meta_pending == 0);
		bfqq->meta_pending--;
	}
}

static int bfq_merge(struct request_queue *q, struct request **req,
		     struct bio *bio)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct request *__rq;

	__rq = bfq_find_rq_fmerge(bfqd, bio);
	if (__rq != NULL && elv_rq_merge_ok(__rq, bio)) {
		*req = __rq;
		return ELEVATOR_FRONT_MERGE;
	}

	return ELEVATOR_NO_MERGE;
}

static void bfq_merged_request(struct request_queue *q, struct request *req,
			       int type)
{
	if (type == ELEVATOR_FRONT_MERGE &&
	    rb_prev(&req->rb_node) &&
	    blk_rq_pos(req) <
	    blk_rq_pos(container_of(rb_prev(&req->rb_node),
				    struct request, rb_node))) {
		struct bfq_queue *bfqq = RQ_BFQQ(req);
		struct bfq_data *bfqd = bfqq->bfqd;
		struct request *prev, *next_rq;

		/* Reposition request in its sort_list */
		elv_rb_del(&bfqq->sort_list, req);
		elv_rb_add(&bfqq->sort_list, req);
		/* Choose next request to be served for bfqq */
		prev = bfqq->next_rq;
		next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req,
					 bfqd->last_position);
		BUG_ON(next_rq == NULL);
		bfqq->next_rq = next_rq;
		/*
		 * If next_rq changes, update both the queue's budget to
		 * fit the new request and the queue's position in its
		 * rq_pos_tree.
		 */
		if (prev != bfqq->next_rq) {
			bfq_updated_next_req(bfqd, bfqq);
			bfq_rq_pos_tree_add(bfqd, bfqq);
		}
	}
}

static void bfq_merged_requests(struct request_queue *q, struct request *rq,
				struct request *next)
{
	struct bfq_queue *bfqq = RQ_BFQQ(rq), *next_bfqq = RQ_BFQQ(next);

	/*
	 * If next and rq belong to the same bfq_queue and next is older
	 * than rq, then reposition rq in the fifo (by substituting next
	 * with rq). Otherwise, if next and rq belong to different
	 * bfq_queues, never reposition rq: in fact, we would have to
	 * reposition it with respect to next's position in its own fifo,
	 * which would most certainly be too expensive with respect to
	 * the benefits.
	 */
	if (bfqq == next_bfqq &&
	    !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
	    time_before(next->fifo_time, rq->fifo_time)) {
		list_del_init(&rq->queuelist);
		list_replace_init(&next->queuelist, &rq->queuelist);
		rq->fifo_time = next->fifo_time;
	}

	if (bfqq->next_rq == next)
		bfqq->next_rq = rq;

	bfq_remove_request(next);
}

/* Must be called with bfqq != NULL */
static inline void bfq_bfqq_end_wr(struct bfq_queue *bfqq)
{
	BUG_ON(bfqq == NULL);
	if (bfq_bfqq_busy(bfqq))
		bfqq->bfqd->wr_busy_queues--;
	bfqq->wr_coeff = 1;
	bfqq->wr_cur_max_time = 0;
	/* Trigger a weight change on the next activation of the queue */
	bfqq->entity.ioprio_changed = 1;
}

static void bfq_end_wr_async_queues(struct bfq_data *bfqd,
				    struct bfq_group *bfqg)
{
	int i, j;

	for (i = 0; i < 2; i++)
		for (j = 0; j < IOPRIO_BE_NR; j++)
			if (bfqg->async_bfqq[i][j] != NULL)
				bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]);
	if (bfqg->async_idle_bfqq != NULL)
		bfq_bfqq_end_wr(bfqg->async_idle_bfqq);
}

static void bfq_end_wr(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq;

	spin_lock_irq(bfqd->queue->queue_lock);

	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list)
		bfq_bfqq_end_wr(bfqq);
	list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list)
		bfq_bfqq_end_wr(bfqq);
	bfq_end_wr_async(bfqd);

	spin_unlock_irq(bfqd->queue->queue_lock);
}

static inline sector_t bfq_io_struct_pos(void *io_struct, bool request)
{
	if (request)
		return blk_rq_pos(io_struct);
	else
		return ((struct bio *)io_struct)->bi_iter.bi_sector;
}

static inline sector_t bfq_dist_from(sector_t pos1,
				     sector_t pos2)
{
	if (pos1 >= pos2)
		return pos1 - pos2;
	else
		return pos2 - pos1;
}

static inline int bfq_rq_close_to_sector(void *io_struct, bool request,
					 sector_t sector)
{
	return bfq_dist_from(bfq_io_struct_pos(io_struct, request), sector) <=
	       BFQQ_SEEK_THR;
}

static struct bfq_queue *bfqq_close(struct bfq_data *bfqd, sector_t sector)
{
	struct rb_root *root = &bfqd->rq_pos_tree;
	struct rb_node *parent, *node;
	struct bfq_queue *__bfqq;

	if (RB_EMPTY_ROOT(root))
		return NULL;

	/*
	 * First, if we find a request starting at the end of the last
	 * request, choose it.
	 */
	__bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL);
	if (__bfqq != NULL)
		return __bfqq;

	/*
	 * If the exact sector wasn't found, the parent of the NULL leaf
	 * will contain the closest sector (rq_pos_tree sorted by
	 * next_request position).
	 */
	__bfqq = rb_entry(parent, struct bfq_queue, pos_node);
	if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector))
		return __bfqq;

	if (blk_rq_pos(__bfqq->next_rq) < sector)
		node = rb_next(&__bfqq->pos_node);
	else
		node = rb_prev(&__bfqq->pos_node);
	if (node == NULL)
		return NULL;

	__bfqq = rb_entry(node, struct bfq_queue, pos_node);
	if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector))
		return __bfqq;

	return NULL;
}

/*
 * bfqd - obvious
 * cur_bfqq - passed in so that we don't decide that the current queue
 *            is closely cooperating with itself
 * sector - used as a reference point to search for a close queue
 */
static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd,
					      struct bfq_queue *cur_bfqq,
					      sector_t sector)
{
	struct bfq_queue *bfqq;

	if (bfq_class_idle(cur_bfqq))
		return NULL;
	if (!bfq_bfqq_sync(cur_bfqq))
		return NULL;
	if (BFQQ_SEEKY(cur_bfqq))
		return NULL;

	/* If device has only one backlogged bfq_queue, don't search. */
	if (bfqd->busy_queues == 1)
		return NULL;

	/*
	 * We should notice if some of the queues are cooperating, e.g.
	 * working closely on the same area of the disk. In that case,
	 * we can group them together and don't waste time idling.
	 */
	bfqq = bfqq_close(bfqd, sector);
	if (bfqq == NULL || bfqq == cur_bfqq)
		return NULL;

	/*
	 * Do not merge queues from different bfq_groups.
	*/
	if (bfqq->entity.parent != cur_bfqq->entity.parent)
		return NULL;

	/*
	 * It only makes sense to merge sync queues.
	 */
	if (!bfq_bfqq_sync(bfqq))
		return NULL;
	if (BFQQ_SEEKY(bfqq))
		return NULL;

	/*
	 * Do not merge queues of different priority classes.
	 */
	if (bfq_class_rt(bfqq) != bfq_class_rt(cur_bfqq))
		return NULL;

	return bfqq;
}

static struct bfq_queue *
bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
{
	int process_refs, new_process_refs;
	struct bfq_queue *__bfqq;

	/*
	 * If there are no process references on the new_bfqq, then it is
	 * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain
	 * may have dropped their last reference (not just their last process
	 * reference).
	 */
	if (!bfqq_process_refs(new_bfqq))
		return NULL;

	/* Avoid a circular list and skip interim queue merges. */
	while ((__bfqq = new_bfqq->new_bfqq)) {
		if (__bfqq == bfqq)
			return NULL;
		new_bfqq = __bfqq;
	}

	process_refs = bfqq_process_refs(bfqq);
	new_process_refs = bfqq_process_refs(new_bfqq);
	/*
	 * If the process for the bfqq has gone away, there is no
	 * sense in merging the queues.
	 */
	if (process_refs == 0 || new_process_refs == 0)
		return NULL;

	bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d",
		new_bfqq->pid);

	/*
	 * Merging is just a redirection: the requests of the process
	 * owning one of the two queues are redirected to the other queue.
	 * The latter queue, in its turn, is set as shared if this is the
	 * first time that the requests of some process are redirected to
	 * it.
	 *
	 * We redirect bfqq to new_bfqq and not the opposite, because we
	 * are in the context of the process owning bfqq, hence we have
	 * the io_cq of this process. So we can immediately configure this
	 * io_cq to redirect the requests of the process to new_bfqq.
	 *
	 * NOTE, even if new_bfqq coincides with the in-service queue, the
	 * io_cq of new_bfqq is not available, because, if the in-service
	 * queue is shared, bfqd->in_service_bic may not point to the
	 * io_cq of the in-service queue.
	 * Redirecting the requests of the process owning bfqq to the
	 * currently in-service queue is in any case the best option, as
	 * we feed the in-service queue with new requests close to the
	 * last request served and, by doing so, hopefully increase the
	 * throughput.
	 */
	bfqq->new_bfqq = new_bfqq;
	atomic_add(process_refs, &new_bfqq->ref);
	return new_bfqq;
}

/*
 * Attempt to schedule a merge of bfqq with the currently in-service queue
 * or with a close queue among the scheduled queues.
 * Return NULL if no merge was scheduled, a pointer to the shared bfq_queue
 * structure otherwise.
 *
 * The OOM queue is not allowed to participate to cooperation: in fact, since
 * the requests temporarily redirected to the OOM queue could be redirected
 * again to dedicated queues at any time, the state needed to correctly
 * handle merging with the OOM queue would be quite complex and expensive
 * to maintain. Besides, in such a critical condition as an out of memory,
 * the benefits of queue merging may be little relevant, or even negligible.
 */
static struct bfq_queue *
bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
		     void *io_struct, bool request)
{
	struct bfq_queue *in_service_bfqq, *new_bfqq;

	if (bfqq->new_bfqq)
		return bfqq->new_bfqq;

	if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq))
		return NULL;

	in_service_bfqq = bfqd->in_service_queue;

	if (in_service_bfqq == NULL || in_service_bfqq == bfqq ||
	    !bfqd->in_service_bic ||
	    unlikely(in_service_bfqq == &bfqd->oom_bfqq))
		goto check_scheduled;

	if (bfq_class_idle(in_service_bfqq) || bfq_class_idle(bfqq))
		goto check_scheduled;

	if (bfq_class_rt(in_service_bfqq) != bfq_class_rt(bfqq))
		goto check_scheduled;

	if (in_service_bfqq->entity.parent != bfqq->entity.parent)
		goto check_scheduled;

	if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) &&
	    bfq_bfqq_sync(in_service_bfqq) && bfq_bfqq_sync(bfqq)) {
		new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq);
		if (new_bfqq != NULL)
			return new_bfqq; /* Merge with in-service queue */
	}

	/*
	 * Check whether there is a cooperator among currently scheduled
	 * queues. The only thing we need is that the bio/request is not
	 * NULL, as we need it to establish whether a cooperator exists.
	 */
check_scheduled:
	new_bfqq = bfq_close_cooperator(bfqd, bfqq,
					bfq_io_struct_pos(io_struct, request));
	if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq))
		return bfq_setup_merge(bfqq, new_bfqq);

	return NULL;
}

static inline void
bfq_bfqq_save_state(struct bfq_queue *bfqq)
{
	/*
	 * If bfqq->bic == NULL, the queue is already shared or its requests
	 * have already been redirected to a shared queue; both idle window
	 * and weight raising state have already been saved. Do nothing.
	 */
	if (bfqq->bic == NULL)
		return;
	if (bfqq->bic->wr_time_left)
		/*
		 * This is the queue of a just-started process, and would
		 * deserve weight raising: we set wr_time_left to the full
		 * weight-raising duration to trigger weight-raising when
		 * and if the queue is split and the first request of the
		 * queue is enqueued.
		 */
		bfqq->bic->wr_time_left = bfq_wr_duration(bfqq->bfqd);
	else if (bfqq->wr_coeff > 1) {
		unsigned long wr_duration =
			jiffies - bfqq->last_wr_start_finish;
		/*
		 * It may happen that a queue's weight raising period lasts
		 * longer than its wr_cur_max_time, as weight raising is
		 * handled only when a request is enqueued or dispatched (it
		 * does not use any timer). If the weight raising period is
		 * about to end, don't save it.
		 */
		if (bfqq->wr_cur_max_time <= wr_duration)
			bfqq->bic->wr_time_left = 0;
		else
			bfqq->bic->wr_time_left =
				bfqq->wr_cur_max_time - wr_duration;
		/*
		 * The bfq_queue is becoming shared or the requests of the
		 * process owning the queue are being redirected to a shared
		 * queue. Stop the weight raising period of the queue, as in
		 * both cases it should not be owned by an interactive or
		 * soft real-time application.
		 */
		bfq_bfqq_end_wr(bfqq);
	} else
		bfqq->bic->wr_time_left = 0;
	bfqq->bic->saved_idle_window = bfq_bfqq_idle_window(bfqq);
	bfqq->bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
	bfqq->bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
	bfqq->bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
	bfqq->bic->cooperations++;
	bfqq->bic->failed_cooperations = 0;
}

static inline void
bfq_get_bic_reference(struct bfq_queue *bfqq)
{
	/*
	 * If bfqq->bic has a non-NULL value, the bic to which it belongs
	 * is about to begin using a shared bfq_queue.
	 */
	if (bfqq->bic)
		atomic_long_inc(&bfqq->bic->icq.ioc->refcount);
}

static void
bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
		struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
{
	bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu",
		(long unsigned)new_bfqq->pid);
	/* Save weight raising and idle window of the merged queues */
	bfq_bfqq_save_state(bfqq);
	bfq_bfqq_save_state(new_bfqq);
	if (bfq_bfqq_IO_bound(bfqq))
		bfq_mark_bfqq_IO_bound(new_bfqq);
	bfq_clear_bfqq_IO_bound(bfqq);
	/*
	 * Grab a reference to the bic, to prevent it from being destroyed
	 * before being possibly touched by a bfq_split_bfqq().
	 */
	bfq_get_bic_reference(bfqq);
	bfq_get_bic_reference(new_bfqq);
	/*
	 * Merge queues (that is, let bic redirect its requests to new_bfqq)
	 */
	bic_set_bfqq(bic, new_bfqq, 1);
	bfq_mark_bfqq_coop(new_bfqq);
	/*
	 * new_bfqq now belongs to at least two bics (it is a shared queue):
	 * set new_bfqq->bic to NULL. bfqq either:
	 * - does not belong to any bic any more, and hence bfqq->bic must
	 *   be set to NULL, or
	 * - is a queue whose owning bics have already been redirected to a
	 *   different queue, hence the queue is destined to not belong to
	 *   any bic soon and bfqq->bic is already NULL (therefore the next
	 *   assignment causes no harm).
	 */
	new_bfqq->bic = NULL;
	bfqq->bic = NULL;
	bfq_put_queue(bfqq);
}

static inline void bfq_bfqq_increase_failed_cooperations(struct bfq_queue *bfqq)
{
	struct bfq_io_cq *bic = bfqq->bic;
	struct bfq_data *bfqd = bfqq->bfqd;

	if (bic && bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh) {
		bic->failed_cooperations++;
		if (bic->failed_cooperations >= bfqd->bfq_failed_cooperations)
			bic->cooperations = 0;
	}
}

static int bfq_allow_merge(struct request_queue *q, struct request *rq,
			   struct bio *bio)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct bfq_io_cq *bic;
	struct bfq_queue *bfqq, *new_bfqq;

	/*
	 * Disallow merge of a sync bio into an async request.
	 */
	if (bfq_bio_sync(bio) && !rq_is_sync(rq))
		return 0;

	/*
	 * Lookup the bfqq that this bio will be queued with. Allow
	 * merge only if rq is queued there.
	 * Queue lock is held here.
	 */
	bic = bfq_bic_lookup(bfqd, current->io_context);
	if (bic == NULL)
		return 0;

	bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio));
	/*
	 * We take advantage of this function to perform an early merge
	 * of the queues of possible cooperating processes.
	 */
	if (bfqq != NULL) {
		new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false);
		if (new_bfqq != NULL) {
			bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq);
			/*
			 * If we get here, the bio will be queued in the
			 * shared queue, i.e., new_bfqq, so use new_bfqq
			 * to decide whether bio and rq can be merged.
			 */
			bfqq = new_bfqq;
		} else
			bfq_bfqq_increase_failed_cooperations(bfqq);
	}

	return bfqq == RQ_BFQQ(rq);
}

static void __bfq_set_in_service_queue(struct bfq_data *bfqd,
				       struct bfq_queue *bfqq)
{
	if (bfqq != NULL) {
		bfq_mark_bfqq_must_alloc(bfqq);
		bfq_mark_bfqq_budget_new(bfqq);
		bfq_clear_bfqq_fifo_expire(bfqq);

		bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8;

		bfq_log_bfqq(bfqd, bfqq,
			     "set_in_service_queue, cur-budget = %lu",
			     bfqq->entity.budget);
	}

	bfqd->in_service_queue = bfqq;
}

/*
 * Get and set a new queue for service.
 */
static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq = bfq_get_next_queue(bfqd);

	__bfq_set_in_service_queue(bfqd, bfqq);
	return bfqq;
}

/*
 * If enough samples have been computed, return the current max budget
 * stored in bfqd, which is dynamically updated according to the
 * estimated disk peak rate; otherwise return the default max budget
 */
static inline unsigned long bfq_max_budget(struct bfq_data *bfqd)
{
	if (bfqd->budgets_assigned < 194)
		return bfq_default_max_budget;
	else
		return bfqd->bfq_max_budget;
}

/*
 * Return min budget, which is a fraction of the current or default
 * max budget (trying with 1/32)
 */
static inline unsigned long bfq_min_budget(struct bfq_data *bfqd)
{
	if (bfqd->budgets_assigned < 194)
		return bfq_default_max_budget / 32;
	else
		return bfqd->bfq_max_budget / 32;
}

static void bfq_arm_slice_timer(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq = bfqd->in_service_queue;
	struct bfq_io_cq *bic;
	unsigned long sl;

	BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list));

	/* Processes have exited, don't wait. */
	bic = bfqd->in_service_bic;
	if (bic == NULL || atomic_read(&bic->icq.ioc->active_ref) == 0)
		return;

	bfq_mark_bfqq_wait_request(bfqq);

	/*
	 * We don't want to idle for seeks, but we do want to allow
	 * fair distribution of slice time for a process doing back-to-back
	 * seeks. So allow a little bit of time for him to submit a new rq.
	 *
	 * To prevent processes with (partly) seeky workloads from
	 * being too ill-treated, grant them a small fraction of the
	 * assigned budget before reducing the waiting time to
	 * BFQ_MIN_TT. This happened to help reduce latency.
	 */
	sl = bfqd->bfq_slice_idle;
	/*
	 * Unless the queue is being weight-raised or the scenario is
	 * asymmetric, grant only minimum idle time if the queue either
	 * has been seeky for long enough or has already proved to be
	 * constantly seeky.
	 */
	if (bfq_sample_valid(bfqq->seek_samples) &&
	    ((BFQQ_SEEKY(bfqq) && bfqq->entity.service >
				  bfq_max_budget(bfqq->bfqd) / 8) ||
	      bfq_bfqq_constantly_seeky(bfqq)) && bfqq->wr_coeff == 1 &&
	    symmetric_scenario)
		sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT));
	else if (bfqq->wr_coeff > 1)
		sl = sl * 3;
	bfqd->last_idling_start = ktime_get();
	mod_timer(&bfqd->idle_slice_timer, jiffies + sl);
	bfq_log(bfqd, "arm idle: %u/%u ms",
		jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle));
}

/*
 * Set the maximum time for the in-service queue to consume its
 * budget. This prevents seeky processes from lowering the disk
 * throughput (always guaranteed with a time slice scheme as in CFQ).
 */
static void bfq_set_budget_timeout(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq = bfqd->in_service_queue;
	unsigned int timeout_coeff;
	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
		timeout_coeff = 1;
	else
		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;

	bfqd->last_budget_start = ktime_get();

	bfq_clear_bfqq_budget_new(bfqq);
	bfqq->budget_timeout = jiffies +
		bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * timeout_coeff;

	bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u",
		jiffies_to_msecs(bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] *
		timeout_coeff));
}

/*
 * Move request from internal lists to the request queue dispatch list.
 */
static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct bfq_queue *bfqq = RQ_BFQQ(rq);

	/*
	 * For consistency, the next instruction should have been executed
	 * after removing the request from the queue and dispatching it.
	 * We execute instead this instruction before bfq_remove_request()
	 * (and hence introduce a temporary inconsistency), for efficiency.
	 * In fact, in a forced_dispatch, this prevents two counters related
	 * to bfqq->dispatched to risk to be uselessly decremented if bfqq
	 * is not in service, and then to be incremented again after
	 * incrementing bfqq->dispatched.
	 */
	bfqq->dispatched++;
	bfq_remove_request(rq);
	elv_dispatch_sort(q, rq);

	if (bfq_bfqq_sync(bfqq))
		bfqd->sync_flight++;
}

/*
 * Return expired entry, or NULL to just start from scratch in rbtree.
 */
static struct request *bfq_check_fifo(struct bfq_queue *bfqq)
{
	struct request *rq = NULL;

	if (bfq_bfqq_fifo_expire(bfqq))
		return NULL;

	bfq_mark_bfqq_fifo_expire(bfqq);

	if (list_empty(&bfqq->fifo))
		return NULL;

	rq = rq_entry_fifo(bfqq->fifo.next);

	if (time_before(jiffies, rq->fifo_time))
		return NULL;

	return rq;
}

static inline unsigned long bfq_bfqq_budget_left(struct bfq_queue *bfqq)
{
	struct bfq_entity *entity = &bfqq->entity;
	return entity->budget - entity->service;
}

static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
{
	BUG_ON(bfqq != bfqd->in_service_queue);

	__bfq_bfqd_reset_in_service(bfqd);

	/*
	 * If this bfqq is shared between multiple processes, check
	 * to make sure that those processes are still issuing I/Os
	 * within the mean seek distance. If not, it may be time to
	 * break the queues apart again.
	 */
	if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq))
		bfq_mark_bfqq_split_coop(bfqq);

	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
		/*
		 * Overloading budget_timeout field to store the time
		 * at which the queue remains with no backlog; used by
		 * the weight-raising mechanism.
		 */
		bfqq->budget_timeout = jiffies;
		bfq_del_bfqq_busy(bfqd, bfqq, 1);
	} else {
		bfq_activate_bfqq(bfqd, bfqq);
		/*
		 * Resort priority tree of potential close cooperators.
		 */
		bfq_rq_pos_tree_add(bfqd, bfqq);
	}
}

/**
 * __bfq_bfqq_recalc_budget - try to adapt the budget to the @bfqq behavior.
 * @bfqd: device data.
 * @bfqq: queue to update.
 * @reason: reason for expiration.
 *
 * Handle the feedback on @bfqq budget.  See the body for detailed
 * comments.
 */
static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
				     struct bfq_queue *bfqq,
				     enum bfqq_expiration reason)
{
	struct request *next_rq;
	unsigned long budget, min_budget;

	budget = bfqq->max_budget;
	min_budget = bfq_min_budget(bfqd);

	BUG_ON(bfqq != bfqd->in_service_queue);

	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %lu, budg left %lu",
		bfqq->entity.budget, bfq_bfqq_budget_left(bfqq));
	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %lu, min budg %lu",
		budget, bfq_min_budget(bfqd));
	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d",
		bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue));

	if (bfq_bfqq_sync(bfqq)) {
		switch (reason) {
		/*
		 * Caveat: in all the following cases we trade latency
		 * for throughput.
		 */
		case BFQ_BFQQ_TOO_IDLE:
			/*
			 * This is the only case where we may reduce
			 * the budget: if there is no request of the
			 * process still waiting for completion, then
			 * we assume (tentatively) that the timer has
			 * expired because the batch of requests of
			 * the process could have been served with a
			 * smaller budget.  Hence, betting that
			 * process will behave in the same way when it
			 * becomes backlogged again, we reduce its
			 * next budget.  As long as we guess right,
			 * this budget cut reduces the latency
			 * experienced by the process.
			 *
			 * However, if there are still outstanding
			 * requests, then the process may have not yet
			 * issued its next request just because it is
			 * still waiting for the completion of some of
			 * the still outstanding ones.  So in this
			 * subcase we do not reduce its budget, on the
			 * contrary we increase it to possibly boost
			 * the throughput, as discussed in the
			 * comments to the BUDGET_TIMEOUT case.
			 */
			if (bfqq->dispatched > 0) /* still outstanding reqs */
				budget = min(budget * 2, bfqd->bfq_max_budget);
			else {
				if (budget > 5 * min_budget)
					budget -= 4 * min_budget;
				else
					budget = min_budget;
			}
			break;
		case BFQ_BFQQ_BUDGET_TIMEOUT:
			/*
			 * We double the budget here because: 1) it
			 * gives the chance to boost the throughput if
			 * this is not a seeky process (which may have
			 * bumped into this timeout because of, e.g.,
			 * ZBR), 2) together with charge_full_budget
			 * it helps give seeky processes higher
			 * timestamps, and hence be served less
			 * frequently.
			 */
			budget = min(budget * 2, bfqd->bfq_max_budget);
			break;
		case BFQ_BFQQ_BUDGET_EXHAUSTED:
			/*
			 * The process still has backlog, and did not
			 * let either the budget timeout or the disk
			 * idling timeout expire. Hence it is not
			 * seeky, has a short thinktime and may be
			 * happy with a higher budget too. So
			 * definitely increase the budget of this good
			 * candidate to boost the disk throughput.
			 */
			budget = min(budget * 4, bfqd->bfq_max_budget);
			break;
		case BFQ_BFQQ_NO_MORE_REQUESTS:
		       /*
			* Leave the budget unchanged.
			*/
		default:
			return;
		}
	} else /* async queue */
	    /* async queues get always the maximum possible budget
	     * (their ability to dispatch is limited by
	     * @bfqd->bfq_max_budget_async_rq).
	     */
		budget = bfqd->bfq_max_budget;

	bfqq->max_budget = budget;

	if (bfqd->budgets_assigned >= 194 && bfqd->bfq_user_max_budget == 0 &&
	    bfqq->max_budget > bfqd->bfq_max_budget)
		bfqq->max_budget = bfqd->bfq_max_budget;

	/*
	 * Make sure that we have enough budget for the next request.
	 * Since the finish time of the bfqq must be kept in sync with
	 * the budget, be sure to call __bfq_bfqq_expire() after the
	 * update.
	 */
	next_rq = bfqq->next_rq;
	if (next_rq != NULL)
		bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget,
					    bfq_serv_to_charge(next_rq, bfqq));
	else
		bfqq->entity.budget = bfqq->max_budget;

	bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %lu",
			next_rq != NULL ? blk_rq_sectors(next_rq) : 0,
			bfqq->entity.budget);
}

static unsigned long bfq_calc_max_budget(u64 peak_rate, u64 timeout)
{
	unsigned long max_budget;

	/*
	 * The max_budget calculated when autotuning is equal to the
	 * amount of sectors transfered in timeout_sync at the
	 * estimated peak rate.
	 */
	max_budget = (unsigned long)(peak_rate * 1000 *
				     timeout >> BFQ_RATE_SHIFT);

	return max_budget;
}

/*
 * In addition to updating the peak rate, checks whether the process
 * is "slow", and returns 1 if so. This slow flag is used, in addition
 * to the budget timeout, to reduce the amount of service provided to
 * seeky processes, and hence reduce their chances to lower the
 * throughput. See the code for more details.
 */
static int bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
				int compensate, enum bfqq_expiration reason)
{
	u64 bw, usecs, expected, timeout;
	ktime_t delta;
	int update = 0;

	if (!bfq_bfqq_sync(bfqq) || bfq_bfqq_budget_new(bfqq))
		return 0;

	if (compensate)
		delta = bfqd->last_idling_start;
	else
		delta = ktime_get();
	delta = ktime_sub(delta, bfqd->last_budget_start);
	usecs = ktime_to_us(delta);

	/* Don't trust short/unrealistic values. */
	if (usecs < 100 || usecs >= LONG_MAX)
		return 0;

	/*
	 * Calculate the bandwidth for the last slice.  We use a 64 bit
	 * value to store the peak rate, in sectors per usec in fixed
	 * point math.  We do so to have enough precision in the estimate
	 * and to avoid overflows.
	 */
	bw = (u64)bfqq->entity.service << BFQ_RATE_SHIFT;
	do_div(bw, (unsigned long)usecs);

	timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]);

	/*
	 * Use only long (> 20ms) intervals to filter out spikes for
	 * the peak rate estimation.
	 */
	if (usecs > 20000) {
		if (bw > bfqd->peak_rate ||
		   (!BFQQ_SEEKY(bfqq) &&
		    reason == BFQ_BFQQ_BUDGET_TIMEOUT)) {
			bfq_log(bfqd, "measured bw =%llu", bw);
			/*
			 * To smooth oscillations use a low-pass filter with
			 * alpha=7/8, i.e.,
			 * new_rate = (7/8) * old_rate + (1/8) * bw
			 */
			do_div(bw, 8);
			if (bw == 0)
				return 0;
			bfqd->peak_rate *= 7;
			do_div(bfqd->peak_rate, 8);
			bfqd->peak_rate += bw;
			update = 1;
			bfq_log(bfqd, "new peak_rate=%llu", bfqd->peak_rate);
		}

		update |= bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES - 1;

		if (bfqd->peak_rate_samples < BFQ_PEAK_RATE_SAMPLES)
			bfqd->peak_rate_samples++;

		if (bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES &&
		    update) {
			int dev_type = blk_queue_nonrot(bfqd->queue);
			if (bfqd->bfq_user_max_budget == 0) {
				bfqd->bfq_max_budget =
					bfq_calc_max_budget(bfqd->peak_rate,
							    timeout);
				bfq_log(bfqd, "new max_budget=%lu",
					bfqd->bfq_max_budget);
			}
			if (bfqd->device_speed == BFQ_BFQD_FAST &&
			    bfqd->peak_rate < device_speed_thresh[dev_type]) {
				bfqd->device_speed = BFQ_BFQD_SLOW;
				bfqd->RT_prod = R_slow[dev_type] *
						T_slow[dev_type];
			} else if (bfqd->device_speed == BFQ_BFQD_SLOW &&
			    bfqd->peak_rate > device_speed_thresh[dev_type]) {
				bfqd->device_speed = BFQ_BFQD_FAST;
				bfqd->RT_prod = R_fast[dev_type] *
						T_fast[dev_type];
			}
		}
	}

	/*
	 * If the process has been served for a too short time
	 * interval to let its possible sequential accesses prevail on
	 * the initial seek time needed to move the disk head on the
	 * first sector it requested, then give the process a chance
	 * and for the moment return false.
	 */
	if (bfqq->entity.budget <= bfq_max_budget(bfqd) / 8)
		return 0;

	/*
	 * A process is considered ``slow'' (i.e., seeky, so that we
	 * cannot treat it fairly in the service domain, as it would
	 * slow down too much the other processes) if, when a slice
	 * ends for whatever reason, it has received service at a
	 * rate that would not be high enough to complete the budget
	 * before the budget timeout expiration.
	 */
	expected = bw * 1000 * timeout >> BFQ_RATE_SHIFT;

	/*
	 * Caveat: processes doing IO in the slower disk zones will
	 * tend to be slow(er) even if not seeky. And the estimated
	 * peak rate will actually be an average over the disk
	 * surface. Hence, to not be too harsh with unlucky processes,
	 * we keep a budget/3 margin of safety before declaring a
	 * process slow.
	 */
	return expected > (4 * bfqq->entity.budget) / 3;
}

/*
 * To be deemed as soft real-time, an application must meet two
 * requirements. First, the application must not require an average
 * bandwidth higher than the approximate bandwidth required to playback or
 * record a compressed high-definition video.
 * The next function is invoked on the completion of the last request of a
 * batch, to compute the next-start time instant, soft_rt_next_start, such
 * that, if the next request of the application does not arrive before
 * soft_rt_next_start, then the above requirement on the bandwidth is met.
 *
 * The second requirement is that the request pattern of the application is
 * isochronous, i.e., that, after issuing a request or a batch of requests,
 * the application stops issuing new requests until all its pending requests
 * have been completed. After that, the application may issue a new batch,
 * and so on.
 * For this reason the next function is invoked to compute
 * soft_rt_next_start only for applications that meet this requirement,
 * whereas soft_rt_next_start is set to infinity for applications that do
 * not.
 *
 * Unfortunately, even a greedy application may happen to behave in an
 * isochronous way if the CPU load is high. In fact, the application may
 * stop issuing requests while the CPUs are busy serving other processes,
 * then restart, then stop again for a while, and so on. In addition, if
 * the disk achieves a low enough throughput with the request pattern
 * issued by the application (e.g., because the request pattern is random
 * and/or the device is slow), then the application may meet the above
 * bandwidth requirement too. To prevent such a greedy application to be
 * deemed as soft real-time, a further rule is used in the computation of
 * soft_rt_next_start: soft_rt_next_start must be higher than the current
 * time plus the maximum time for which the arrival of a request is waited
 * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle.
 * This filters out greedy applications, as the latter issue instead their
 * next request as soon as possible after the last one has been completed
 * (in contrast, when a batch of requests is completed, a soft real-time
 * application spends some time processing data).
 *
 * Unfortunately, the last filter may easily generate false positives if
 * only bfqd->bfq_slice_idle is used as a reference time interval and one
 * or both the following cases occur:
 * 1) HZ is so low that the duration of a jiffy is comparable to or higher
 *    than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with
 *    HZ=100.
 * 2) jiffies, instead of increasing at a constant rate, may stop increasing
 *    for a while, then suddenly 'jump' by several units to recover the lost
 *    increments. This seems to happen, e.g., inside virtual machines.
 * To address this issue, we do not use as a reference time interval just
 * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In
 * particular we add the minimum number of jiffies for which the filter
 * seems to be quite precise also in embedded systems and KVM/QEMU virtual
 * machines.
 */
static inline unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
						       struct bfq_queue *bfqq)
{
	return max(bfqq->last_idle_bklogged +
		   HZ * bfqq->service_from_backlogged /
		   bfqd->bfq_wr_max_softrt_rate,
		   jiffies + bfqq->bfqd->bfq_slice_idle + 4);
}

/*
 * Return the largest-possible time instant such that, for as long as possible,
 * the current time will be lower than this time instant according to the macro
 * time_is_before_jiffies().
 */
static inline unsigned long bfq_infinity_from_now(unsigned long now)
{
	return now + ULONG_MAX / 2;
}

/**
 * bfq_bfqq_expire - expire a queue.
 * @bfqd: device owning the queue.
 * @bfqq: the queue to expire.
 * @compensate: if true, compensate for the time spent idling.
 * @reason: the reason causing the expiration.
 *
 *
 * If the process associated to the queue is slow (i.e., seeky), or in
 * case of budget timeout, or, finally, if it is async, we
 * artificially charge it an entire budget (independently of the
 * actual service it received). As a consequence, the queue will get
 * higher timestamps than the correct ones upon reactivation, and
 * hence it will be rescheduled as if it had received more service
 * than what it actually received. In the end, this class of processes
 * will receive less service in proportion to how slowly they consume
 * their budgets (and hence how seriously they tend to lower the
 * throughput).
 *
 * In contrast, when a queue expires because it has been idling for
 * too much or because it exhausted its budget, we do not touch the
 * amount of service it has received. Hence when the queue will be
 * reactivated and its timestamps updated, the latter will be in sync
 * with the actual service received by the queue until expiration.
 *
 * Charging a full budget to the first type of queues and the exact
 * service to the others has the effect of using the WF2Q+ policy to
 * schedule the former on a timeslice basis, without violating the
 * service domain guarantees of the latter.
 */
static void bfq_bfqq_expire(struct bfq_data *bfqd,
			    struct bfq_queue *bfqq,
			    int compensate,
			    enum bfqq_expiration reason)
{
	int slow;
	BUG_ON(bfqq != bfqd->in_service_queue);

	/* Update disk peak rate for autotuning and check whether the
	 * process is slow (see bfq_update_peak_rate).
	 */
	slow = bfq_update_peak_rate(bfqd, bfqq, compensate, reason);

	/*
	 * As above explained, 'punish' slow (i.e., seeky), timed-out
	 * and async queues, to favor sequential sync workloads.
	 *
	 * Processes doing I/O in the slower disk zones will tend to be
	 * slow(er) even if not seeky. Hence, since the estimated peak
	 * rate is actually an average over the disk surface, these
	 * processes may timeout just for bad luck. To avoid punishing
	 * them we do not charge a full budget to a process that
	 * succeeded in consuming at least 2/3 of its budget.
	 */
	if (slow || (reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
		     bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3))
		bfq_bfqq_charge_full_budget(bfqq);

	bfqq->service_from_backlogged += bfqq->entity.service;

	if (BFQQ_SEEKY(bfqq) && reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
	    !bfq_bfqq_constantly_seeky(bfqq)) {
		bfq_mark_bfqq_constantly_seeky(bfqq);
		if (!blk_queue_nonrot(bfqd->queue))
			bfqd->const_seeky_busy_in_flight_queues++;
	}

	if (reason == BFQ_BFQQ_TOO_IDLE &&
	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10 )
		bfq_clear_bfqq_IO_bound(bfqq);

	if (bfqd->low_latency && bfqq->wr_coeff == 1)
		bfqq->last_wr_start_finish = jiffies;

	if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 &&
	    RB_EMPTY_ROOT(&bfqq->sort_list)) {
		/*
		 * If we get here, and there are no outstanding requests,
		 * then the request pattern is isochronous (see the comments
		 * to the function bfq_bfqq_softrt_next_start()). Hence we
		 * can compute soft_rt_next_start. If, instead, the queue
		 * still has outstanding requests, then we have to wait
		 * for the completion of all the outstanding requests to
		 * discover whether the request pattern is actually
		 * isochronous.
		 */
		if (bfqq->dispatched == 0)
			bfqq->soft_rt_next_start =
				bfq_bfqq_softrt_next_start(bfqd, bfqq);
		else {
			/*
			 * The application is still waiting for the
			 * completion of one or more requests:
			 * prevent it from possibly being incorrectly
			 * deemed as soft real-time by setting its
			 * soft_rt_next_start to infinity. In fact,
			 * without this assignment, the application
			 * would be incorrectly deemed as soft
			 * real-time if:
			 * 1) it issued a new request before the
			 *    completion of all its in-flight
			 *    requests, and
			 * 2) at that time, its soft_rt_next_start
			 *    happened to be in the past.
			 */
			bfqq->soft_rt_next_start =
				bfq_infinity_from_now(jiffies);
			/*
			 * Schedule an update of soft_rt_next_start to when
			 * the task may be discovered to be isochronous.
			 */
			bfq_mark_bfqq_softrt_update(bfqq);
		}
	}

	bfq_log_bfqq(bfqd, bfqq,
		"expire (%d, slow %d, num_disp %d, idle_win %d)", reason,
		slow, bfqq->dispatched, bfq_bfqq_idle_window(bfqq));

	/*
	 * Increase, decrease or leave budget unchanged according to
	 * reason.
	 */
	__bfq_bfqq_recalc_budget(bfqd, bfqq, reason);
	__bfq_bfqq_expire(bfqd, bfqq);
}

/*
 * Budget timeout is not implemented through a dedicated timer, but
 * just checked on request arrivals and completions, as well as on
 * idle timer expirations.
 */
static int bfq_bfqq_budget_timeout(struct bfq_queue *bfqq)
{
	if (bfq_bfqq_budget_new(bfqq) ||
	    time_before(jiffies, bfqq->budget_timeout))
		return 0;
	return 1;
}

/*
 * If we expire a queue that is waiting for the arrival of a new
 * request, we may prevent the fictitious timestamp back-shifting that
 * allows the guarantees of the queue to be preserved (see [1] for
 * this tricky aspect). Hence we return true only if this condition
 * does not hold, or if the queue is slow enough to deserve only to be
 * kicked off for preserving a high throughput.
*/
static inline int bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)
{
	bfq_log_bfqq(bfqq->bfqd, bfqq,
		"may_budget_timeout: wait_request %d left %d timeout %d",
		bfq_bfqq_wait_request(bfqq),
			bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3,
		bfq_bfqq_budget_timeout(bfqq));

	return (!bfq_bfqq_wait_request(bfqq) ||
		bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3)
		&&
		bfq_bfqq_budget_timeout(bfqq);
}

/*
 * Device idling is allowed only for the queues for which this function
 * returns true. For this reason, the return value of this function plays a
 * critical role for both throughput boosting and service guarantees. The
 * return value is computed through a logical expression. In this rather
 * long comment, we try to briefly describe all the details and motivations
 * behind the components of this logical expression.
 *
 * First, the expression is false if bfqq is not sync, or if: bfqq happened
 * to become active during a large burst of queue activations, and the
 * pattern of requests bfqq contains boosts the throughput if bfqq is
 * expired. In fact, queues that became active during a large burst benefit
 * only from throughput, as discussed in the comments to bfq_handle_burst.
 * In this respect, expiring bfqq certainly boosts the throughput on NCQ-
 * capable flash-based devices, whereas, on rotational devices, it boosts
 * the throughput only if bfqq contains random requests.
 *
 * On the opposite end, if (a) bfqq is sync, (b) the above burst-related
 * condition does not hold, and (c) bfqq is being weight-raised, then the
 * expression always evaluates to true, as device idling is instrumental
 * for preserving low-latency guarantees (see [1]). If, instead, conditions
 * (a) and (b) do hold, but (c) does not, then the expression evaluates to
 * true only if: (1) bfqq is I/O-bound and has a non-null idle window, and
 * (2) at least one of the following two conditions holds.
 * The first condition is that the device is not performing NCQ, because
 * idling the device most certainly boosts the throughput if this condition
 * holds and bfqq is I/O-bound and has been granted a non-null idle window.
 * The second compound condition is made of the logical AND of two components.
 *
 * The first component is true only if there is no weight-raised busy
 * queue. This guarantees that the device is not idled for a sync non-
 * weight-raised queue when there are busy weight-raised queues. The former
 * is then expired immediately if empty. Combined with the timestamping
 * rules of BFQ (see [1] for details), this causes sync non-weight-raised
 * queues to get a lower number of requests served, and hence to ask for a
 * lower number of requests from the request pool, before the busy weight-
 * raised queues get served again.
 *
 * This is beneficial for the processes associated with weight-raised
 * queues, when the request pool is saturated (e.g., in the presence of
 * write hogs). In fact, if the processes associated with the other queues
 * ask for requests at a lower rate, then weight-raised processes have a
 * higher probability to get a request from the pool immediately (or at
 * least soon) when they need one. Hence they have a higher probability to
 * actually get a fraction of the disk throughput proportional to their
 * high weight. This is especially true with NCQ-capable drives, which
 * enqueue several requests in advance and further reorder internally-
 * queued requests.
 *
 * In the end, mistreating non-weight-raised queues when there are busy
 * weight-raised queues seems to mitigate starvation problems in the
 * presence of heavy write workloads and NCQ, and hence to guarantee a
 * higher application and system responsiveness in these hostile scenarios.
 *
 * If the first component of the compound condition is instead true, i.e.,
 * there is no weight-raised busy queue, then the second component of the
 * compound condition takes into account service-guarantee and throughput
 * issues related to NCQ (recall that the compound condition is evaluated
 * only if the device is detected as supporting NCQ).
 *
 * As for service guarantees, allowing the drive to enqueue more than one
 * request at a time, and hence delegating de facto final scheduling
 * decisions to the drive's internal scheduler, causes loss of control on
 * the actual request service order. In this respect, when the drive is
 * allowed to enqueue more than one request at a time, the service
 * distribution enforced by the drive's internal scheduler is likely to
 * coincide with the desired device-throughput distribution only in the
 * following, perfectly symmetric, scenario:
 * 1) all active queues have the same weight,
 * 2) all active groups at the same level in the groups tree have the same
 *    weight,
 * 3) all active groups at the same level in the groups tree have the same
 *    number of children.
 *
 * Even in such a scenario, sequential I/O may still receive a preferential
 * treatment, but this is not likely to be a big issue with flash-based
 * devices, because of their non-dramatic loss of throughput with random
 * I/O. Things do differ with HDDs, for which additional care is taken, as
 * explained after completing the discussion for flash-based devices.
 *
 * Unfortunately, keeping the necessary state for evaluating exactly the
 * above symmetry conditions would be quite complex and time-consuming.
 * Therefore BFQ evaluates instead the following stronger sub-conditions,
 * for which it is much easier to maintain the needed state:
 * 1) all active queues have the same weight,
 * 2) all active groups have the same weight,
 * 3) all active groups have at most one active child each.
 * In particular, the last two conditions are always true if hierarchical
 * support and the cgroups interface are not enabled, hence no state needs
 * to be maintained in this case.
 *
 * According to the above considerations, the second component of the
 * compound condition evaluates to true if any of the above symmetry
 * sub-condition does not hold, or the device is not flash-based. Therefore,
 * if also the first component is true, then idling is allowed for a sync
 * queue. These are the only sub-conditions considered if the device is
 * flash-based, as, for such a device, it is sensible to force idling only
 * for service-guarantee issues. In fact, as for throughput, idling
 * NCQ-capable flash-based devices would not boost the throughput even
 * with sequential I/O; rather it would lower the throughput in proportion
 * to how fast the device is. In the end, (only) if all the three
 * sub-conditions hold and the device is flash-based, the compound
 * condition evaluates to false and therefore no idling is performed.
 *
 * As already said, things change with a rotational device, where idling
 * boosts the throughput with sequential I/O (even with NCQ). Hence, for
 * such a device the second component of the compound condition evaluates
 * to true also if the following additional sub-condition does not hold:
 * the queue is constantly seeky. Unfortunately, this different behavior
 * with respect to flash-based devices causes an additional asymmetry: if
 * some sync queues enjoy idling and some other sync queues do not, then
 * the latter get a low share of the device throughput, simply because the
 * former get many requests served after being set as in service, whereas
 * the latter do not. As a consequence, to guarantee the desired throughput
 * distribution, on HDDs the compound expression evaluates to true (and
 * hence device idling is performed) also if the following last symmetry
 * condition does not hold: no other queue is benefiting from idling. Also
 * this last condition is actually replaced with a simpler-to-maintain and
 * stronger condition: there is no busy queue which is not constantly seeky
 * (and hence may also benefit from idling).
 *
 * To sum up, when all the required symmetry and throughput-boosting
 * sub-conditions hold, the second component of the compound condition
 * evaluates to false, and hence no idling is performed. This helps to
 * keep the drives' internal queues full on NCQ-capable devices, and hence
 * to boost the throughput, without causing 'almost' any loss of service
 * guarantees. The 'almost' follows from the fact that, if the internal
 * queue of one such device is filled while all the sub-conditions hold,
 * but at some point in time some sub-condition stops to hold, then it may
 * become impossible to let requests be served in the new desired order
 * until all the requests already queued in the device have been served.
 */
static inline bool bfq_bfqq_must_not_expire(struct bfq_queue *bfqq)
{
	struct bfq_data *bfqd = bfqq->bfqd;
#define cond_for_seeky_on_ncq_hdd (bfq_bfqq_constantly_seeky(bfqq) && \
				   bfqd->busy_in_flight_queues == \
				   bfqd->const_seeky_busy_in_flight_queues)

#define cond_for_expiring_in_burst	(bfq_bfqq_in_large_burst(bfqq) && \
					 bfqd->hw_tag && \
					 (blk_queue_nonrot(bfqd->queue) || \
					  bfq_bfqq_constantly_seeky(bfqq)))

/*
 * Condition for expiring a non-weight-raised queue (and hence not idling
 * the device).
 */
#define cond_for_expiring_non_wr  (bfqd->hw_tag && \
				   (bfqd->wr_busy_queues > 0 || \
				    (blk_queue_nonrot(bfqd->queue) || \
				      cond_for_seeky_on_ncq_hdd)))

	return bfq_bfqq_sync(bfqq) &&
		!cond_for_expiring_in_burst &&
		(bfqq->wr_coeff > 1 || !symmetric_scenario ||
		 (bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_idle_window(bfqq) &&
		  !cond_for_expiring_non_wr)
	);
}

/*
 * If the in-service queue is empty but sync, and the function
 * bfq_bfqq_must_not_expire returns true, then:
 * 1) the queue must remain in service and cannot be expired, and
 * 2) the disk must be idled to wait for the possible arrival of a new
 *    request for the queue.
 * See the comments to the function bfq_bfqq_must_not_expire for the reasons
 * why performing device idling is the best choice to boost the throughput
 * and preserve service guarantees when bfq_bfqq_must_not_expire itself
 * returns true.
 */
static inline bool bfq_bfqq_must_idle(struct bfq_queue *bfqq)
{
	struct bfq_data *bfqd = bfqq->bfqd;

	return RB_EMPTY_ROOT(&bfqq->sort_list) && bfqd->bfq_slice_idle != 0 &&
	       bfq_bfqq_must_not_expire(bfqq);
}

/*
 * Select a queue for service.  If we have a current queue in service,
 * check whether to continue servicing it, or retrieve and set a new one.
 */
static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq;
	struct request *next_rq;
	enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT;

	bfqq = bfqd->in_service_queue;
	if (bfqq == NULL)
		goto new_queue;

	bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue");

	if (bfq_may_expire_for_budg_timeout(bfqq) &&
	    !timer_pending(&bfqd->idle_slice_timer) &&
	    !bfq_bfqq_must_idle(bfqq))
		goto expire;

	next_rq = bfqq->next_rq;
	/*
	 * If bfqq has requests queued and it has enough budget left to
	 * serve them, keep the queue, otherwise expire it.
	 */
	if (next_rq != NULL) {
		if (bfq_serv_to_charge(next_rq, bfqq) >
			bfq_bfqq_budget_left(bfqq)) {
			reason = BFQ_BFQQ_BUDGET_EXHAUSTED;
			goto expire;
		} else {
			/*
			 * The idle timer may be pending because we may
			 * not disable disk idling even when a new request
			 * arrives.
			 */
			if (timer_pending(&bfqd->idle_slice_timer)) {
				/*
				 * If we get here: 1) at least a new request
				 * has arrived but we have not disabled the
				 * timer because the request was too small,
				 * 2) then the block layer has unplugged
				 * the device, causing the dispatch to be
				 * invoked.
				 *
				 * Since the device is unplugged, now the
				 * requests are probably large enough to
				 * provide a reasonable throughput.
				 * So we disable idling.
				 */
				bfq_clear_bfqq_wait_request(bfqq);
				del_timer(&bfqd->idle_slice_timer);
			}
			goto keep_queue;
		}
	}

	/*
	 * No requests pending. However, if the in-service queue is idling
	 * for a new request, or has requests waiting for a completion and
	 * may idle after their completion, then keep it anyway.
	 */
	if (timer_pending(&bfqd->idle_slice_timer) ||
	    (bfqq->dispatched != 0 && bfq_bfqq_must_not_expire(bfqq))) {
		bfqq = NULL;
		goto keep_queue;
	}

	reason = BFQ_BFQQ_NO_MORE_REQUESTS;
expire:
	bfq_bfqq_expire(bfqd, bfqq, 0, reason);
new_queue:
	bfqq = bfq_set_in_service_queue(bfqd);
	bfq_log(bfqd, "select_queue: new queue %d returned",
		bfqq != NULL ? bfqq->pid : 0);
keep_queue:
	return bfqq;
}

static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
{
	struct bfq_entity *entity = &bfqq->entity;
	if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */
		bfq_log_bfqq(bfqd, bfqq,
			"raising period dur %u/%u msec, old coeff %u, w %d(%d)",
			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),
			jiffies_to_msecs(bfqq->wr_cur_max_time),
			bfqq->wr_coeff,
			bfqq->entity.weight, bfqq->entity.orig_weight);

		BUG_ON(bfqq != bfqd->in_service_queue && entity->weight !=
		       entity->orig_weight * bfqq->wr_coeff);
		if (entity->ioprio_changed)
			bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change");

		/*
		 * If the queue was activated in a burst, or
		 * too much time has elapsed from the beginning
		 * of this weight-raising period, or the queue has
		 * exceeded the acceptable number of cooperations,
		 * then end weight raising.
		 */
		if (bfq_bfqq_in_large_burst(bfqq) ||
		    bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh ||
		    time_is_before_jiffies(bfqq->last_wr_start_finish +
					   bfqq->wr_cur_max_time)) {
			bfqq->last_wr_start_finish = jiffies;
			bfq_log_bfqq(bfqd, bfqq,
				     "wrais ending at %lu, rais_max_time %u",
				     bfqq->last_wr_start_finish,
				     jiffies_to_msecs(bfqq->wr_cur_max_time));
			bfq_bfqq_end_wr(bfqq);
		}
	}
	/* Update weight both if it must be raised and if it must be lowered */
	if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1))
		__bfq_entity_update_weight_prio(
			bfq_entity_service_tree(entity),
			entity);
}

/*
 * Dispatch one request from bfqq, moving it to the request queue
 * dispatch list.
 */
static int bfq_dispatch_request(struct bfq_data *bfqd,
				struct bfq_queue *bfqq)
{
	int dispatched = 0;
	struct request *rq;
	unsigned long service_to_charge;

	BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list));

	/* Follow expired path, else get first next available. */
	rq = bfq_check_fifo(bfqq);
	if (rq == NULL)
		rq = bfqq->next_rq;
	service_to_charge = bfq_serv_to_charge(rq, bfqq);

	if (service_to_charge > bfq_bfqq_budget_left(bfqq)) {
		/*
		 * This may happen if the next rq is chosen in fifo order
		 * instead of sector order. The budget is properly
		 * dimensioned to be always sufficient to serve the next
		 * request only if it is chosen in sector order. The reason
		 * is that it would be quite inefficient and little useful
		 * to always make sure that the budget is large enough to
		 * serve even the possible next rq in fifo order.
		 * In fact, requests are seldom served in fifo order.
		 *
		 * Expire the queue for budget exhaustion, and make sure
		 * that the next act_budget is enough to serve the next
		 * request, even if it comes from the fifo expired path.
		 */
		bfqq->next_rq = rq;
		/*
		 * Since this dispatch is failed, make sure that
		 * a new one will be performed
		 */
		if (!bfqd->rq_in_driver)
			bfq_schedule_dispatch(bfqd);
		goto expire;
	}

	/* Finally, insert request into driver dispatch list. */
	bfq_bfqq_served(bfqq, service_to_charge);
	bfq_dispatch_insert(bfqd->queue, rq);

	bfq_update_wr_data(bfqd, bfqq);

	bfq_log_bfqq(bfqd, bfqq,
			"dispatched %u sec req (%llu), budg left %lu",
			blk_rq_sectors(rq),
			(long long unsigned)blk_rq_pos(rq),
			bfq_bfqq_budget_left(bfqq));

	dispatched++;

	if (bfqd->in_service_bic == NULL) {
		atomic_long_inc(&RQ_BIC(rq)->icq.ioc->refcount);
		bfqd->in_service_bic = RQ_BIC(rq);
	}

	if (bfqd->busy_queues > 1 && ((!bfq_bfqq_sync(bfqq) &&
	    dispatched >= bfqd->bfq_max_budget_async_rq) ||
	    bfq_class_idle(bfqq)))
		goto expire;

	return dispatched;

expire:
	bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_EXHAUSTED);
	return dispatched;
}

static int __bfq_forced_dispatch_bfqq(struct bfq_queue *bfqq)
{
	int dispatched = 0;

	while (bfqq->next_rq != NULL) {
		bfq_dispatch_insert(bfqq->bfqd->queue, bfqq->next_rq);
		dispatched++;
	}

	BUG_ON(!list_empty(&bfqq->fifo));
	return dispatched;
}

/*
 * Drain our current requests.
 * Used for barriers and when switching io schedulers on-the-fly.
 */
static int bfq_forced_dispatch(struct bfq_data *bfqd)
{
	struct bfq_queue *bfqq, *n;
	struct bfq_service_tree *st;
	int dispatched = 0;

	bfqq = bfqd->in_service_queue;
	if (bfqq != NULL)
		__bfq_bfqq_expire(bfqd, bfqq);

	/*
	 * Loop through classes, and be careful to leave the scheduler
	 * in a consistent state, as feedback mechanisms and vtime
	 * updates cannot be disabled during the process.
	 */
	list_for_each_entry_safe(bfqq, n, &bfqd->active_list, bfqq_list) {
		st = bfq_entity_service_tree(&bfqq->entity);

		dispatched += __bfq_forced_dispatch_bfqq(bfqq);
		bfqq->max_budget = bfq_max_budget(bfqd);

		bfq_forget_idle(st);
	}

	BUG_ON(bfqd->busy_queues != 0);

	return dispatched;
}

static int bfq_dispatch_requests(struct request_queue *q, int force)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct bfq_queue *bfqq;
	int max_dispatch;

	bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues);
	if (bfqd->busy_queues == 0)
		return 0;

	if (unlikely(force))
		return bfq_forced_dispatch(bfqd);

	bfqq = bfq_select_queue(bfqd);
	if (bfqq == NULL)
		return 0;

	if (bfq_class_idle(bfqq))
		max_dispatch = 1;

	if (!bfq_bfqq_sync(bfqq))
		max_dispatch = bfqd->bfq_max_budget_async_rq;

	if (!bfq_bfqq_sync(bfqq) && bfqq->dispatched >= max_dispatch) {
		if (bfqd->busy_queues > 1)
			return 0;
		if (bfqq->dispatched >= 4 * max_dispatch)
			return 0;
	}

	if (bfqd->sync_flight != 0 && !bfq_bfqq_sync(bfqq))
		return 0;

	bfq_clear_bfqq_wait_request(bfqq);
	BUG_ON(timer_pending(&bfqd->idle_slice_timer));

	if (!bfq_dispatch_request(bfqd, bfqq))
		return 0;

	bfq_log_bfqq(bfqd, bfqq, "dispatched %s request",
			bfq_bfqq_sync(bfqq) ? "sync" : "async");

	return 1;
}

/*
 * Task holds one reference to the queue, dropped when task exits.  Each rq
 * in-flight on this queue also holds a reference, dropped when rq is freed.
 *
 * Queue lock must be held here.
 */
static void bfq_put_queue(struct bfq_queue *bfqq)
{
	struct bfq_data *bfqd = bfqq->bfqd;

	BUG_ON(atomic_read(&bfqq->ref) <= 0);

	bfq_log_bfqq(bfqd, bfqq, "put_queue: %p %d", bfqq,
		     atomic_read(&bfqq->ref));
	if (!atomic_dec_and_test(&bfqq->ref))
		return;

	BUG_ON(rb_first(&bfqq->sort_list) != NULL);
	BUG_ON(bfqq->allocated[READ] + bfqq->allocated[WRITE] != 0);
	BUG_ON(bfqq->entity.tree != NULL);
	BUG_ON(bfq_bfqq_busy(bfqq));
	BUG_ON(bfqd->in_service_queue == bfqq);

	if (bfq_bfqq_sync(bfqq))
		/*
		 * The fact that this queue is being destroyed does not
		 * invalidate the fact that this queue may have been
		 * activated during the current burst. As a consequence,
		 * although the queue does not exist anymore, and hence
		 * needs to be removed from the burst list if there,
		 * the burst size has not to be decremented.
		 */
		hlist_del_init(&bfqq->burst_list_node);

	bfq_log_bfqq(bfqd, bfqq, "put_queue: %p freed", bfqq);

	kmem_cache_free(bfq_pool, bfqq);
}

static void bfq_put_cooperator(struct bfq_queue *bfqq)
{
	struct bfq_queue *__bfqq, *next;

	/*
	 * If this queue was scheduled to merge with another queue, be
	 * sure to drop the reference taken on that queue (and others in
	 * the merge chain). See bfq_setup_merge and bfq_merge_bfqqs.
	 */
	__bfqq = bfqq->new_bfqq;
	while (__bfqq) {
		if (__bfqq == bfqq)
			break;
		next = __bfqq->new_bfqq;
		bfq_put_queue(__bfqq);
		__bfqq = next;
	}
}

static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
{
	if (bfqq == bfqd->in_service_queue) {
		__bfq_bfqq_expire(bfqd, bfqq);
		bfq_schedule_dispatch(bfqd);
	}

	bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq,
		     atomic_read(&bfqq->ref));

	bfq_put_cooperator(bfqq);

	bfq_put_queue(bfqq);
}

static inline void bfq_init_icq(struct io_cq *icq)
{
	struct bfq_io_cq *bic = icq_to_bic(icq);

	bic->ttime.last_end_request = jiffies;
	/*
	 * A newly created bic indicates that the process has just
	 * started doing I/O, and is probably mapping into memory its
	 * executable and libraries: it definitely needs weight raising.
	 * There is however the possibility that the process performs,
	 * for a while, I/O close to some other process. EQM intercepts
	 * this behavior and may merge the queue corresponding to the
	 * process  with some other queue, BEFORE the weight of the queue
	 * is raised. Merged queues are not weight-raised (they are assumed
	 * to belong to processes that benefit only from high throughput).
	 * If the merge is basically the consequence of an accident, then
	 * the queue will be split soon and will get back its old weight.
	 * It is then important to write down somewhere that this queue
	 * does need weight raising, even if it did not make it to get its
	 * weight raised before being merged. To this purpose, we overload
	 * the field raising_time_left and assign 1 to it, to mark the queue
	 * as needing weight raising.
	 */
	bic->wr_time_left = 1;
}

static void bfq_exit_icq(struct io_cq *icq)
{
	struct bfq_io_cq *bic = icq_to_bic(icq);
	struct bfq_data *bfqd = bic_to_bfqd(bic);

	if (bic->bfqq[BLK_RW_ASYNC]) {
		bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_ASYNC]);
		bic->bfqq[BLK_RW_ASYNC] = NULL;
	}

	if (bic->bfqq[BLK_RW_SYNC]) {
		/*
		 * If the bic is using a shared queue, put the reference
		 * taken on the io_context when the bic started using a
		 * shared bfq_queue.
		 */
		if (bfq_bfqq_coop(bic->bfqq[BLK_RW_SYNC]))
			put_io_context(icq->ioc);
		bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_SYNC]);
		bic->bfqq[BLK_RW_SYNC] = NULL;
	}
}

/*
 * Update the entity prio values; note that the new values will not
 * be used until the next (re)activation.
 */
static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
{
	struct task_struct *tsk = current;
	int ioprio_class;

	ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);
	switch (ioprio_class) {
	default:
		dev_err(bfqq->bfqd->queue->backing_dev_info.dev,
			"bfq: bad prio class %d\n", ioprio_class);
	case IOPRIO_CLASS_NONE:
		/*
		 * No prio set, inherit CPU scheduling settings.
		 */
		bfqq->entity.new_ioprio = task_nice_ioprio(tsk);
		bfqq->entity.new_ioprio_class = task_nice_ioclass(tsk);
		break;
	case IOPRIO_CLASS_RT:
		bfqq->entity.new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);
		bfqq->entity.new_ioprio_class = IOPRIO_CLASS_RT;
		break;
	case IOPRIO_CLASS_BE:
		bfqq->entity.new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);
		bfqq->entity.new_ioprio_class = IOPRIO_CLASS_BE;
		break;
	case IOPRIO_CLASS_IDLE:
		bfqq->entity.new_ioprio_class = IOPRIO_CLASS_IDLE;
		bfqq->entity.new_ioprio = 7;
		bfq_clear_bfqq_idle_window(bfqq);
		break;
	}

	if (bfqq->entity.new_ioprio < 0 ||
	    bfqq->entity.new_ioprio >= IOPRIO_BE_NR) {
		printk(KERN_CRIT "bfq_set_next_ioprio_data: new_ioprio %d\n",
				 bfqq->entity.new_ioprio);
		BUG();
	}

	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->entity.new_ioprio);
	bfqq->entity.ioprio_changed = 1;
}

static void bfq_check_ioprio_change(struct bfq_io_cq *bic)
{
	struct bfq_data *bfqd;
	struct bfq_queue *bfqq, *new_bfqq;
	struct bfq_group *bfqg;
	unsigned long uninitialized_var(flags);
	int ioprio = bic->icq.ioc->ioprio;

	bfqd = bfq_get_bfqd_locked(&(bic->icq.q->elevator->elevator_data),
				   &flags);
	/*
	 * This condition may trigger on a newly created bic, be sure to
	 * drop the lock before returning.
	 */
	if (unlikely(bfqd == NULL) || likely(bic->ioprio == ioprio))
		goto out;

	bic->ioprio = ioprio;

	bfqq = bic->bfqq[BLK_RW_ASYNC];
	if (bfqq != NULL) {
		bfqg = container_of(bfqq->entity.sched_data, struct bfq_group,
				    sched_data);
		new_bfqq = bfq_get_queue(bfqd, bfqg, BLK_RW_ASYNC, bic,
					 GFP_ATOMIC);
		if (new_bfqq != NULL) {
			bic->bfqq[BLK_RW_ASYNC] = new_bfqq;
			bfq_log_bfqq(bfqd, bfqq,
				     "check_ioprio_change: bfqq %p %d",
				     bfqq, atomic_read(&bfqq->ref));
			bfq_put_queue(bfqq);
		}
	}

	bfqq = bic->bfqq[BLK_RW_SYNC];
	if (bfqq != NULL)
		bfq_set_next_ioprio_data(bfqq, bic);

out:
	bfq_put_bfqd_unlock(bfqd, &flags);
}

static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
			  struct bfq_io_cq *bic, pid_t pid, int is_sync)
{
	RB_CLEAR_NODE(&bfqq->entity.rb_node);
	INIT_LIST_HEAD(&bfqq->fifo);
	INIT_HLIST_NODE(&bfqq->burst_list_node);

	atomic_set(&bfqq->ref, 0);
	bfqq->bfqd = bfqd;

	if (bic)
		bfq_set_next_ioprio_data(bfqq, bic);

	if (is_sync) {
		if (!bfq_class_idle(bfqq))
			bfq_mark_bfqq_idle_window(bfqq);
		bfq_mark_bfqq_sync(bfqq);
	}
	bfq_mark_bfqq_IO_bound(bfqq);

	/* Tentative initial value to trade off between thr and lat */
	bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3;
	bfqq->pid = pid;

	bfqq->wr_coeff = 1;
	bfqq->last_wr_start_finish = 0;
	/*
	 * Set to the value for which bfqq will not be deemed as
	 * soft rt when it becomes backlogged.
	 */
	bfqq->soft_rt_next_start = bfq_infinity_from_now(jiffies);
}

static struct bfq_queue *bfq_find_alloc_queue(struct bfq_data *bfqd,
					      struct bfq_group *bfqg,
					      int is_sync,
					      struct bfq_io_cq *bic,
					      gfp_t gfp_mask)
{
	struct bfq_queue *bfqq, *new_bfqq = NULL;

retry:
	/* bic always exists here */
	bfqq = bic_to_bfqq(bic, is_sync);

	/*
	 * Always try a new alloc if we fall back to the OOM bfqq
	 * originally, since it should just be a temporary situation.
	 */
	if (bfqq == NULL || bfqq == &bfqd->oom_bfqq) {
		bfqq = NULL;
		if (new_bfqq != NULL) {
			bfqq = new_bfqq;
			new_bfqq = NULL;
		} else if (gfp_mask & __GFP_WAIT) {
			spin_unlock_irq(bfqd->queue->queue_lock);
			new_bfqq = kmem_cache_alloc_node(bfq_pool,
					gfp_mask | __GFP_ZERO,
					bfqd->queue->node);
			spin_lock_irq(bfqd->queue->queue_lock);
			if (new_bfqq != NULL)
				goto retry;
		} else {
			bfqq = kmem_cache_alloc_node(bfq_pool,
					gfp_mask | __GFP_ZERO,
					bfqd->queue->node);
		}

		if (bfqq != NULL) {
			bfq_init_bfqq(bfqd, bfqq, bic, current->pid,
                                      is_sync);
			bfq_init_entity(&bfqq->entity, bfqg);
			bfq_log_bfqq(bfqd, bfqq, "allocated");
		} else {
			bfqq = &bfqd->oom_bfqq;
			bfq_log_bfqq(bfqd, bfqq, "using oom bfqq");
		}
	}

	if (new_bfqq != NULL)
		kmem_cache_free(bfq_pool, new_bfqq);

	return bfqq;
}

static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
					       struct bfq_group *bfqg,
					       int ioprio_class, int ioprio)
{
	switch (ioprio_class) {
	case IOPRIO_CLASS_RT:
		return &bfqg->async_bfqq[0][ioprio];
	case IOPRIO_CLASS_NONE:
		ioprio = IOPRIO_NORM;
		/* fall through */
	case IOPRIO_CLASS_BE:
		return &bfqg->async_bfqq[1][ioprio];
	case IOPRIO_CLASS_IDLE:
		return &bfqg->async_idle_bfqq;
	default:
		BUG();
	}
}

static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
				       struct bfq_group *bfqg, int is_sync,
				       struct bfq_io_cq *bic, gfp_t gfp_mask)
{
	const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio);
	const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);
	struct bfq_queue **async_bfqq = NULL;
	struct bfq_queue *bfqq = NULL;

	if (!is_sync) {
		async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class,
						  ioprio);
		bfqq = *async_bfqq;
	}

	if (bfqq == NULL)
		bfqq = bfq_find_alloc_queue(bfqd, bfqg, is_sync, bic, gfp_mask);

	/*
	 * Pin the queue now that it's allocated, scheduler exit will
	 * prune it.
	 */
	if (!is_sync && *async_bfqq == NULL) {
		atomic_inc(&bfqq->ref);
		bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d",
			     bfqq, atomic_read(&bfqq->ref));
		*async_bfqq = bfqq;
	}

	atomic_inc(&bfqq->ref);
	bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq,
		     atomic_read(&bfqq->ref));
	return bfqq;
}

static void bfq_update_io_thinktime(struct bfq_data *bfqd,
				    struct bfq_io_cq *bic)
{
	unsigned long elapsed = jiffies - bic->ttime.last_end_request;
	unsigned long ttime = min(elapsed, 2UL * bfqd->bfq_slice_idle);

	bic->ttime.ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8;
	bic->ttime.ttime_total = (7*bic->ttime.ttime_total + 256*ttime) / 8;
	bic->ttime.ttime_mean = (bic->ttime.ttime_total + 128) /
				bic->ttime.ttime_samples;
}

static void bfq_update_io_seektime(struct bfq_data *bfqd,
				   struct bfq_queue *bfqq,
				   struct request *rq)
{
	sector_t sdist;
	u64 total;

	if (bfqq->last_request_pos < blk_rq_pos(rq))
		sdist = blk_rq_pos(rq) - bfqq->last_request_pos;
	else
		sdist = bfqq->last_request_pos - blk_rq_pos(rq);

	/*
	 * Don't allow the seek distance to get too large from the
	 * odd fragment, pagein, etc.
	 */
	if (bfqq->seek_samples == 0) /* first request, not really a seek */
		sdist = 0;
	else if (bfqq->seek_samples <= 60) /* second & third seek */
		sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*1024);
	else
		sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*64);

	bfqq->seek_samples = (7*bfqq->seek_samples + 256) / 8;
	bfqq->seek_total = (7*bfqq->seek_total + (u64)256*sdist) / 8;
	total = bfqq->seek_total + (bfqq->seek_samples/2);
	do_div(total, bfqq->seek_samples);
	bfqq->seek_mean = (sector_t)total;

	bfq_log_bfqq(bfqd, bfqq, "dist=%llu mean=%llu", (u64)sdist,
			(u64)bfqq->seek_mean);
}

/*
 * Disable idle window if the process thinks too long or seeks so much that
 * it doesn't matter.
 */
static void bfq_update_idle_window(struct bfq_data *bfqd,
				   struct bfq_queue *bfqq,
				   struct bfq_io_cq *bic)
{
	int enable_idle;

	/* Don't idle for async or idle io prio class. */
	if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq))
		return;

	/* Idle window just restored, statistics are meaningless. */
	if (bfq_bfqq_just_split(bfqq))
		return;

	enable_idle = bfq_bfqq_idle_window(bfqq);

	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
	    bfqd->bfq_slice_idle == 0 ||
		(bfqd->hw_tag && BFQQ_SEEKY(bfqq) &&
			bfqq->wr_coeff == 1))
		enable_idle = 0;
	else if (bfq_sample_valid(bic->ttime.ttime_samples)) {
		if (bic->ttime.ttime_mean > bfqd->bfq_slice_idle &&
			bfqq->wr_coeff == 1)
			enable_idle = 0;
		else
			enable_idle = 1;
	}
	bfq_log_bfqq(bfqd, bfqq, "update_idle_window: enable_idle %d",
		enable_idle);

	if (enable_idle)
		bfq_mark_bfqq_idle_window(bfqq);
	else
		bfq_clear_bfqq_idle_window(bfqq);
}

/*
 * Called when a new fs request (rq) is added to bfqq.  Check if there's
 * something we should do about it.
 */
static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
			    struct request *rq)
{
	struct bfq_io_cq *bic = RQ_BIC(rq);

	if (rq->cmd_flags & REQ_META)
		bfqq->meta_pending++;

	bfq_update_io_thinktime(bfqd, bic);
	bfq_update_io_seektime(bfqd, bfqq, rq);
	if (!BFQQ_SEEKY(bfqq) && bfq_bfqq_constantly_seeky(bfqq)) {
		bfq_clear_bfqq_constantly_seeky(bfqq);
		if (!blk_queue_nonrot(bfqd->queue)) {
			BUG_ON(!bfqd->const_seeky_busy_in_flight_queues);
			bfqd->const_seeky_busy_in_flight_queues--;
		}
	}
	if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 ||
	    !BFQQ_SEEKY(bfqq))
		bfq_update_idle_window(bfqd, bfqq, bic);
	bfq_clear_bfqq_just_split(bfqq);

	bfq_log_bfqq(bfqd, bfqq,
		     "rq_enqueued: idle_window=%d (seeky %d, mean %llu)",
		     bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq),
		     (long long unsigned)bfqq->seek_mean);

	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);

	if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) {
		int small_req = bfqq->queued[rq_is_sync(rq)] == 1 &&
				blk_rq_sectors(rq) < 32;
		int budget_timeout = bfq_bfqq_budget_timeout(bfqq);

		/*
		 * There is just this request queued: if the request
		 * is small and the queue is not to be expired, then
		 * just exit.
		 *
		 * In this way, if the disk is being idled to wait for
		 * a new request from the in-service queue, we avoid
		 * unplugging the device and committing the disk to serve
		 * just a small request. On the contrary, we wait for
		 * the block layer to decide when to unplug the device:
		 * hopefully, new requests will be merged to this one
		 * quickly, then the device will be unplugged and
		 * larger requests will be dispatched.
		 */
		if (small_req && !budget_timeout)
			return;

		/*
		 * A large enough request arrived, or the queue is to
		 * be expired: in both cases disk idling is to be
		 * stopped, so clear wait_request flag and reset
		 * timer.
		 */
		bfq_clear_bfqq_wait_request(bfqq);
		del_timer(&bfqd->idle_slice_timer);

		/*
		 * The queue is not empty, because a new request just
		 * arrived. Hence we can safely expire the queue, in
		 * case of budget timeout, without risking that the
		 * timestamps of the queue are not updated correctly.
		 * See [1] for more details.
		 */
		if (budget_timeout)
			bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_TIMEOUT);

		/*
		 * Let the request rip immediately, or let a new queue be
		 * selected if bfqq has just been expired.
		 */
		__blk_run_queue(bfqd->queue);
	}
}

static void bfq_insert_request(struct request_queue *q, struct request *rq)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq;

	assert_spin_locked(bfqd->queue->queue_lock);

	/*
	 * An unplug may trigger a requeue of a request from the device
	 * driver: make sure we are in process context while trying to
	 * merge two bfq_queues.
	 */
	if (!in_interrupt()) {
		new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true);
		if (new_bfqq != NULL) {
			if (bic_to_bfqq(RQ_BIC(rq), 1) != bfqq)
				new_bfqq = bic_to_bfqq(RQ_BIC(rq), 1);
			/*
			 * Release the request's reference to the old bfqq
			 * and make sure one is taken to the shared queue.
			 */
			new_bfqq->allocated[rq_data_dir(rq)]++;
			bfqq->allocated[rq_data_dir(rq)]--;
			atomic_inc(&new_bfqq->ref);
			bfq_put_queue(bfqq);
			if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq)
				bfq_merge_bfqqs(bfqd, RQ_BIC(rq),
						bfqq, new_bfqq);
			rq->elv.priv[1] = new_bfqq;
			bfqq = new_bfqq;
		} else
			bfq_bfqq_increase_failed_cooperations(bfqq);
	}

	bfq_add_request(rq);

	/*
	 * Here a newly-created bfq_queue has already started a weight-raising
	 * period: clear raising_time_left to prevent bfq_bfqq_save_state()
	 * from assigning it a full weight-raising period. See the detailed
	 * comments about this field in bfq_init_icq().
	 */
	if (bfqq->bic != NULL)
		bfqq->bic->wr_time_left = 0;
	rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
	list_add_tail(&rq->queuelist, &bfqq->fifo);

	bfq_rq_enqueued(bfqd, bfqq, rq);
}

static void bfq_update_hw_tag(struct bfq_data *bfqd)
{
	bfqd->max_rq_in_driver = max(bfqd->max_rq_in_driver,
				     bfqd->rq_in_driver);

	if (bfqd->hw_tag == 1)
		return;

	/*
	 * This sample is valid if the number of outstanding requests
	 * is large enough to allow a queueing behavior.  Note that the
	 * sum is not exact, as it's not taking into account deactivated
	 * requests.
	 */
	if (bfqd->rq_in_driver + bfqd->queued < BFQ_HW_QUEUE_THRESHOLD)
		return;

	if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES)
		return;

	bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD;
	bfqd->max_rq_in_driver = 0;
	bfqd->hw_tag_samples = 0;
}

static void bfq_completed_request(struct request_queue *q, struct request *rq)
{
	struct bfq_queue *bfqq = RQ_BFQQ(rq);
	struct bfq_data *bfqd = bfqq->bfqd;
	bool sync = bfq_bfqq_sync(bfqq);

	bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left (%d)",
		     blk_rq_sectors(rq), sync);

	bfq_update_hw_tag(bfqd);

	BUG_ON(!bfqd->rq_in_driver);
	BUG_ON(!bfqq->dispatched);
	bfqd->rq_in_driver--;
	bfqq->dispatched--;

	if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) {
		bfq_weights_tree_remove(bfqd, &bfqq->entity,
					&bfqd->queue_weights_tree);
		if (!blk_queue_nonrot(bfqd->queue)) {
			BUG_ON(!bfqd->busy_in_flight_queues);
			bfqd->busy_in_flight_queues--;
			if (bfq_bfqq_constantly_seeky(bfqq)) {
				BUG_ON(!bfqd->
					const_seeky_busy_in_flight_queues);
				bfqd->const_seeky_busy_in_flight_queues--;
			}
		}
	}

	if (sync) {
		bfqd->sync_flight--;
		RQ_BIC(rq)->ttime.last_end_request = jiffies;
	}

	/*
	 * If we are waiting to discover whether the request pattern of the
	 * task associated with the queue is actually isochronous, and
	 * both requisites for this condition to hold are satisfied, then
	 * compute soft_rt_next_start (see the comments to the function
	 * bfq_bfqq_softrt_next_start()).
	 */
	if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 &&
	    RB_EMPTY_ROOT(&bfqq->sort_list))
		bfqq->soft_rt_next_start =
			bfq_bfqq_softrt_next_start(bfqd, bfqq);

	/*
	 * If this is the in-service queue, check if it needs to be expired,
	 * or if we want to idle in case it has no pending requests.
	 */
	if (bfqd->in_service_queue == bfqq) {
		if (bfq_bfqq_budget_new(bfqq))
			bfq_set_budget_timeout(bfqd);

		if (bfq_bfqq_must_idle(bfqq)) {
			bfq_arm_slice_timer(bfqd);
			goto out;
		} else if (bfq_may_expire_for_budg_timeout(bfqq))
			bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_TIMEOUT);
		else if (RB_EMPTY_ROOT(&bfqq->sort_list) &&
			 (bfqq->dispatched == 0 ||
			  !bfq_bfqq_must_not_expire(bfqq)))
			bfq_bfqq_expire(bfqd, bfqq, 0,
					BFQ_BFQQ_NO_MORE_REQUESTS);
	}

	if (!bfqd->rq_in_driver)
		bfq_schedule_dispatch(bfqd);

out:
	return;
}

static inline int __bfq_may_queue(struct bfq_queue *bfqq)
{
	if (bfq_bfqq_wait_request(bfqq) && bfq_bfqq_must_alloc(bfqq)) {
		bfq_clear_bfqq_must_alloc(bfqq);
		return ELV_MQUEUE_MUST;
	}

	return ELV_MQUEUE_MAY;
}

static int bfq_may_queue(struct request_queue *q, int rw)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct task_struct *tsk = current;
	struct bfq_io_cq *bic;
	struct bfq_queue *bfqq;

	/*
	 * Don't force setup of a queue from here, as a call to may_queue
	 * does not necessarily imply that a request actually will be
	 * queued. So just lookup a possibly existing queue, or return
	 * 'may queue' if that fails.
	 */
	bic = bfq_bic_lookup(bfqd, tsk->io_context);
	if (bic == NULL)
		return ELV_MQUEUE_MAY;

	bfqq = bic_to_bfqq(bic, rw_is_sync(rw));
	if (bfqq != NULL)
		return __bfq_may_queue(bfqq);

	return ELV_MQUEUE_MAY;
}

/*
 * Queue lock held here.
 */
static void bfq_put_request(struct request *rq)
{
	struct bfq_queue *bfqq = RQ_BFQQ(rq);

	if (bfqq != NULL) {
		const int rw = rq_data_dir(rq);

		BUG_ON(!bfqq->allocated[rw]);
		bfqq->allocated[rw]--;

		rq->elv.priv[0] = NULL;
		rq->elv.priv[1] = NULL;

		bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d",
			     bfqq, atomic_read(&bfqq->ref));
		bfq_put_queue(bfqq);
	}
}

/*
 * Returns NULL if a new bfqq should be allocated, or the old bfqq if this
 * was the last process referring to said bfqq.
 */
static struct bfq_queue *
bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
{
	bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue");

	put_io_context(bic->icq.ioc);

	if (bfqq_process_refs(bfqq) == 1) {
		bfqq->pid = current->pid;
		bfq_clear_bfqq_coop(bfqq);
		bfq_clear_bfqq_split_coop(bfqq);
		return bfqq;
	}

	bic_set_bfqq(bic, NULL, 1);

	bfq_put_cooperator(bfqq);

	bfq_put_queue(bfqq);
	return NULL;
}

/*
 * Allocate bfq data structures associated with this request.
 */
static int bfq_set_request(struct request_queue *q, struct request *rq,
			   struct bio *bio, gfp_t gfp_mask)
{
	struct bfq_data *bfqd = q->elevator->elevator_data;
	struct bfq_io_cq *bic = icq_to_bic(rq->elv.icq);
	const int rw = rq_data_dir(rq);
	const int is_sync = rq_is_sync(rq);
	struct bfq_queue *bfqq;
	struct bfq_group *bfqg;
	unsigned long flags;
	bool split = false;

	might_sleep_if(gfp_mask & __GFP_WAIT);

	bfq_check_ioprio_change(bic);

	spin_lock_irqsave(q->queue_lock, flags);

	if (bic == NULL)
		goto queue_fail;

	bfqg = bfq_bic_update_cgroup(bic);

new_queue:
	bfqq = bic_to_bfqq(bic, is_sync);
	if (bfqq == NULL || bfqq == &bfqd->oom_bfqq) {
		bfqq = bfq_get_queue(bfqd, bfqg, is_sync, bic, gfp_mask);
		bic_set_bfqq(bic, bfqq, is_sync);
		if (split && is_sync) {
			if ((bic->was_in_burst_list && bfqd->large_burst) ||
			    bic->saved_in_large_burst)
				bfq_mark_bfqq_in_large_burst(bfqq);
			else {
			    bfq_clear_bfqq_in_large_burst(bfqq);
			    if (bic->was_in_burst_list)
			       hlist_add_head(&bfqq->burst_list_node,
				              &bfqd->burst_list);
			}
		}
	} else {
		/* If the queue was seeky for too long, break it apart. */
		if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) {
			bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq");
			bfqq = bfq_split_bfqq(bic, bfqq);
			split = true;
			if (!bfqq)
				goto new_queue;
		}
	}

	bfqq->allocated[rw]++;
	atomic_inc(&bfqq->ref);
	bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq,
		     atomic_read(&bfqq->ref));

	rq->elv.priv[0] = bic;
	rq->elv.priv[1] = bfqq;

	/*
	 * If a bfq_queue has only one process reference, it is owned
	 * by only one bfq_io_cq: we can set the bic field of the
	 * bfq_queue to the address of that structure. Also, if the
	 * queue has just been split, mark a flag so that the
	 * information is available to the other scheduler hooks.
	 */
	if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {
		bfqq->bic = bic;
		if (split) {
			bfq_mark_bfqq_just_split(bfqq);
			/*
			 * If the queue has just been split from a shared
			 * queue, restore the idle window and the possible
			 * weight raising period.
			 */
			bfq_bfqq_resume_state(bfqq, bic);
		}
	}

	spin_unlock_irqrestore(q->queue_lock, flags);

	return 0;

queue_fail:
	bfq_schedule_dispatch(bfqd);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return 1;
}

static void bfq_kick_queue(struct work_struct *work)
{
	struct bfq_data *bfqd =
		container_of(work, struct bfq_data, unplug_work);
	struct request_queue *q = bfqd->queue;

	spin_lock_irq(q->queue_lock);
	__blk_run_queue(q);
	spin_unlock_irq(q->queue_lock);
}

/*
 * Handler of the expiration of the timer running if the in-service queue
 * is idling inside its time slice.
 */
static void bfq_idle_slice_timer(unsigned long data)
{
	struct bfq_data *bfqd = (struct bfq_data *)data;
	struct bfq_queue *bfqq;
	unsigned long flags;
	enum bfqq_expiration reason;

	spin_lock_irqsave(bfqd->queue->queue_lock, flags);

	bfqq = bfqd->in_service_queue;
	/*
	 * Theoretical race here: the in-service queue can be NULL or
	 * different from the queue that was idling if the timer handler
	 * spins on the queue_lock and a new request arrives for the
	 * current queue and there is a full dispatch cycle that changes
	 * the in-service queue.  This can hardly happen, but in the worst
	 * case we just expire a queue too early.
	 */
	if (bfqq != NULL) {
		bfq_log_bfqq(bfqd, bfqq, "slice_timer expired");
		if (bfq_bfqq_budget_timeout(bfqq))
			/*
			 * Also here the queue can be safely expired
			 * for budget timeout without wasting
			 * guarantees
			 */
			reason = BFQ_BFQQ_BUDGET_TIMEOUT;
		else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0)
			/*
			 * The queue may not be empty upon timer expiration,
			 * because we may not disable the timer when the
			 * first request of the in-service queue arrives
			 * during disk idling.
			 */
			reason = BFQ_BFQQ_TOO_IDLE;
		else
			goto schedule_dispatch;

		bfq_bfqq_expire(bfqd, bfqq, 1, reason);
	}

schedule_dispatch:
	bfq_schedule_dispatch(bfqd);

	spin_unlock_irqrestore(bfqd->queue->queue_lock, flags);
}

static void bfq_shutdown_timer_wq(struct bfq_data *bfqd)
{
	del_timer_sync(&bfqd->idle_slice_timer);
	cancel_work_sync(&bfqd->unplug_work);
}

static inline void __bfq_put_async_bfqq(struct bfq_data *bfqd,
					struct bfq_queue **bfqq_ptr)
{
	struct bfq_group *root_group = bfqd->root_group;
	struct bfq_queue *bfqq = *bfqq_ptr;

	bfq_log(bfqd, "put_async_bfqq: %p", bfqq);
	if (bfqq != NULL) {
		bfq_bfqq_move(bfqd, bfqq, &bfqq->entity, root_group);
		bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d",
			     bfqq, atomic_read(&bfqq->ref));
		bfq_put_queue(bfqq);
		*bfqq_ptr = NULL;
	}
}

/*
 * Release all the bfqg references to its async queues.  If we are
 * deallocating the group these queues may still contain requests, so
 * we reparent them to the root cgroup (i.e., the only one that will
 * exist for sure until all the requests on a device are gone).
 */
static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
{
	int i, j;

	for (i = 0; i < 2; i++)
		for (j = 0; j < IOPRIO_BE_NR; j++)
			__bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]);

	__bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq);
}

static void bfq_exit_queue(struct elevator_queue *e)
{
	struct bfq_data *bfqd = e->elevator_data;
	struct request_queue *q = bfqd->queue;
	struct bfq_queue *bfqq, *n;

	bfq_shutdown_timer_wq(bfqd);

	spin_lock_irq(q->queue_lock);

	BUG_ON(bfqd->in_service_queue != NULL);
	list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list)
		bfq_deactivate_bfqq(bfqd, bfqq, 0);

	bfq_disconnect_groups(bfqd);
	spin_unlock_irq(q->queue_lock);

	bfq_shutdown_timer_wq(bfqd);

	synchronize_rcu();

	BUG_ON(timer_pending(&bfqd->idle_slice_timer));

	bfq_free_root_group(bfqd);
	kfree(bfqd);
}

static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
{
	struct bfq_group *bfqg;
	struct bfq_data *bfqd;
	struct elevator_queue *eq;

	eq = elevator_alloc(q, e);
	if (eq == NULL)
		return -ENOMEM;

	bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node);
	if (bfqd == NULL) {
		kobject_put(&eq->kobj);
		return -ENOMEM;
	}
	eq->elevator_data = bfqd;

	/*
	 * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues.
	 * Grab a permanent reference to it, so that the normal code flow
	 * will not attempt to free it.
	 */
	bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0);
	atomic_inc(&bfqd->oom_bfqq.ref);
	bfqd->oom_bfqq.entity.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO;
	bfqd->oom_bfqq.entity.new_ioprio_class = IOPRIO_CLASS_BE;
	bfqd->oom_bfqq.entity.new_weight =
		bfq_ioprio_to_weight(bfqd->oom_bfqq.entity.new_ioprio);
	/*
	 * Trigger weight initialization, according to ioprio, at the
	 * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio
	 * class won't be changed any more.
	 */
	bfqd->oom_bfqq.entity.ioprio_changed = 1;

	bfqd->queue = q;

	spin_lock_irq(q->queue_lock);
	q->elevator = eq;
	spin_unlock_irq(q->queue_lock);

	bfqg = bfq_alloc_root_group(bfqd, q->node);
	if (bfqg == NULL) {
		kfree(bfqd);
		kobject_put(&eq->kobj);
		return -ENOMEM;
	}

	bfqd->root_group = bfqg;
	bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group);
#ifdef CONFIG_CGROUP_BFQIO
	bfqd->active_numerous_groups = 0;
#endif

	init_timer(&bfqd->idle_slice_timer);
	bfqd->idle_slice_timer.function = bfq_idle_slice_timer;
	bfqd->idle_slice_timer.data = (unsigned long)bfqd;

	bfqd->rq_pos_tree = RB_ROOT;
	bfqd->queue_weights_tree = RB_ROOT;
	bfqd->group_weights_tree = RB_ROOT;

	INIT_WORK(&bfqd->unplug_work, bfq_kick_queue);

	INIT_LIST_HEAD(&bfqd->active_list);
	INIT_LIST_HEAD(&bfqd->idle_list);
	INIT_HLIST_HEAD(&bfqd->burst_list);

	bfqd->hw_tag = -1;

	bfqd->bfq_max_budget = bfq_default_max_budget;

	bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0];
	bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1];
	bfqd->bfq_back_max = bfq_back_max;
	bfqd->bfq_back_penalty = bfq_back_penalty;
	bfqd->bfq_slice_idle = bfq_slice_idle;
	bfqd->bfq_class_idle_last_service = 0;
	bfqd->bfq_max_budget_async_rq = bfq_max_budget_async_rq;
	bfqd->bfq_timeout[BLK_RW_ASYNC] = bfq_timeout_async;
	bfqd->bfq_timeout[BLK_RW_SYNC] = bfq_timeout_sync;

	bfqd->bfq_coop_thresh = 2;
	bfqd->bfq_failed_cooperations = 7000;
	bfqd->bfq_requests_within_timer = 120;

	bfqd->bfq_large_burst_thresh = 11;
	bfqd->bfq_burst_interval = msecs_to_jiffies(500);

	bfqd->low_latency = true;

	bfqd->bfq_wr_coeff = 20;
	bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300);
	bfqd->bfq_wr_max_time = 0;
	bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000);
	bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500);
	bfqd->bfq_wr_max_softrt_rate = 7000; /*
					      * Approximate rate required
					      * to playback or record a
					      * high-definition compressed
					      * video.
					      */
	bfqd->wr_busy_queues = 0;
	bfqd->busy_in_flight_queues = 0;
	bfqd->const_seeky_busy_in_flight_queues = 0;

	/*
	 * Begin by assuming, optimistically, that the device peak rate is
	 * equal to the highest reference rate.
	 */
	bfqd->RT_prod = R_fast[blk_queue_nonrot(bfqd->queue)] *
			T_fast[blk_queue_nonrot(bfqd->queue)];
	bfqd->peak_rate = R_fast[blk_queue_nonrot(bfqd->queue)];
	bfqd->device_speed = BFQ_BFQD_FAST;

	return 0;
}

static void bfq_slab_kill(void)
{
	if (bfq_pool != NULL)
		kmem_cache_destroy(bfq_pool);
}

static int __init bfq_slab_setup(void)
{
	bfq_pool = KMEM_CACHE(bfq_queue, 0);
	if (bfq_pool == NULL)
		return -ENOMEM;
	return 0;
}

static ssize_t bfq_var_show(unsigned int var, char *page)
{
	return sprintf(page, "%d\n", var);
}

static ssize_t bfq_var_store(unsigned long *var, const char *page,
			     size_t count)
{
	unsigned long new_val;
	int ret = kstrtoul(page, 10, &new_val);

	if (ret == 0)
		*var = new_val;

	return count;
}

static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page)
{
	struct bfq_data *bfqd = e->elevator_data;
	return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ?
		       jiffies_to_msecs(bfqd->bfq_wr_max_time) :
		       jiffies_to_msecs(bfq_wr_duration(bfqd)));
}

static ssize_t bfq_weights_show(struct elevator_queue *e, char *page)
{
	struct bfq_queue *bfqq;
	struct bfq_data *bfqd = e->elevator_data;
	ssize_t num_char = 0;

	num_char += sprintf(page + num_char, "Tot reqs queued %d\n\n",
			    bfqd->queued);

	spin_lock_irq(bfqd->queue->queue_lock);

	num_char += sprintf(page + num_char, "Active:\n");
	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) {
	  num_char += sprintf(page + num_char,
			      "pid%d: weight %hu, nr_queued %d %d, dur %d/%u\n",
			      bfqq->pid,
			      bfqq->entity.weight,
			      bfqq->queued[0],
			      bfqq->queued[1],
			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),
			jiffies_to_msecs(bfqq->wr_cur_max_time));
	}

	num_char += sprintf(page + num_char, "Idle:\n");
	list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) {
			num_char += sprintf(page + num_char,
				"pid%d: weight %hu, dur %d/%u\n",
				bfqq->pid,
				bfqq->entity.weight,
				jiffies_to_msecs(jiffies -
					bfqq->last_wr_start_finish),
				jiffies_to_msecs(bfqq->wr_cur_max_time));
	}

	spin_unlock_irq(bfqd->queue->queue_lock);

	return num_char;
}

#define SHOW_FUNCTION(__FUNC, __VAR, __CONV)				\
static ssize_t __FUNC(struct elevator_queue *e, char *page)		\
{									\
	struct bfq_data *bfqd = e->elevator_data;			\
	unsigned int __data = __VAR;					\
	if (__CONV)							\
		__data = jiffies_to_msecs(__data);			\
	return bfq_var_show(__data, (page));				\
}
SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 1);
SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 1);
SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0);
SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1);
SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0);
SHOW_FUNCTION(bfq_max_budget_async_rq_show,
	      bfqd->bfq_max_budget_async_rq, 0);
SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout[BLK_RW_SYNC], 1);
SHOW_FUNCTION(bfq_timeout_async_show, bfqd->bfq_timeout[BLK_RW_ASYNC], 1);
SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0);
SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0);
SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1);
SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, 1);
SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async,
	1);
SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0);
#undef SHOW_FUNCTION

#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)			\
static ssize_t								\
__FUNC(struct elevator_queue *e, const char *page, size_t count)	\
{									\
	struct bfq_data *bfqd = e->elevator_data;			\
	unsigned long uninitialized_var(__data);			\
	int ret = bfq_var_store(&__data, (page), count);		\
	if (__data < (MIN))						\
		__data = (MIN);						\
	else if (__data > (MAX))					\
		__data = (MAX);						\
	if (__CONV)							\
		*(__PTR) = msecs_to_jiffies(__data);			\
	else								\
		*(__PTR) = __data;					\
	return ret;							\
}
STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1,
		INT_MAX, 1);
STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1,
		INT_MAX, 1);
STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1,
		INT_MAX, 0);
STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1);
STORE_FUNCTION(bfq_max_budget_async_rq_store, &bfqd->bfq_max_budget_async_rq,
		1, INT_MAX, 0);
STORE_FUNCTION(bfq_timeout_async_store, &bfqd->bfq_timeout[BLK_RW_ASYNC], 0,
		INT_MAX, 1);
STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0);
STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1);
STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX,
		1);
STORE_FUNCTION(bfq_wr_min_idle_time_store, &bfqd->bfq_wr_min_idle_time, 0,
		INT_MAX, 1);
STORE_FUNCTION(bfq_wr_min_inter_arr_async_store,
		&bfqd->bfq_wr_min_inter_arr_async, 0, INT_MAX, 1);
STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0,
		INT_MAX, 0);
#undef STORE_FUNCTION

/* do nothing for the moment */
static ssize_t bfq_weights_store(struct elevator_queue *e,
				    const char *page, size_t count)
{
	return count;
}

static inline unsigned long bfq_estimated_max_budget(struct bfq_data *bfqd)
{
	u64 timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]);

	if (bfqd->peak_rate_samples >= BFQ_PEAK_RATE_SAMPLES)
		return bfq_calc_max_budget(bfqd->peak_rate, timeout);
	else
		return bfq_default_max_budget;
}

static ssize_t bfq_max_budget_store(struct elevator_queue *e,
				    const char *page, size_t count)
{
	struct bfq_data *bfqd = e->elevator_data;
	unsigned long uninitialized_var(__data);
	int ret = bfq_var_store(&__data, (page), count);

	if (__data == 0)
		bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd);
	else {
		if (__data > INT_MAX)
			__data = INT_MAX;
		bfqd->bfq_max_budget = __data;
	}

	bfqd->bfq_user_max_budget = __data;

	return ret;
}

static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
				      const char *page, size_t count)
{
	struct bfq_data *bfqd = e->elevator_data;
	unsigned long uninitialized_var(__data);
	int ret = bfq_var_store(&__data, (page), count);

	if (__data < 1)
		__data = 1;
	else if (__data > INT_MAX)
		__data = INT_MAX;

	bfqd->bfq_timeout[BLK_RW_SYNC] = msecs_to_jiffies(__data);
	if (bfqd->bfq_user_max_budget == 0)
		bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd);

	return ret;
}

static ssize_t bfq_low_latency_store(struct elevator_queue *e,
				     const char *page, size_t count)
{
	struct bfq_data *bfqd = e->elevator_data;
	unsigned long uninitialized_var(__data);
	int ret = bfq_var_store(&__data, (page), count);

	if (__data > 1)
		__data = 1;
	if (__data == 0 && bfqd->low_latency != 0)
		bfq_end_wr(bfqd);
	bfqd->low_latency = __data;

	return ret;
}

#define BFQ_ATTR(name) \
	__ATTR(name, S_IRUGO|S_IWUSR, bfq_##name##_show, bfq_##name##_store)

static struct elv_fs_entry bfq_attrs[] = {
	BFQ_ATTR(fifo_expire_sync),
	BFQ_ATTR(fifo_expire_async),
	BFQ_ATTR(back_seek_max),
	BFQ_ATTR(back_seek_penalty),
	BFQ_ATTR(slice_idle),
	BFQ_ATTR(max_budget),
	BFQ_ATTR(max_budget_async_rq),
	BFQ_ATTR(timeout_sync),
	BFQ_ATTR(timeout_async),
	BFQ_ATTR(low_latency),
	BFQ_ATTR(wr_coeff),
	BFQ_ATTR(wr_max_time),
	BFQ_ATTR(wr_rt_max_time),
	BFQ_ATTR(wr_min_idle_time),
	BFQ_ATTR(wr_min_inter_arr_async),
	BFQ_ATTR(wr_max_softrt_rate),
	BFQ_ATTR(weights),
	__ATTR_NULL
};

static struct elevator_type iosched_bfq = {
	.ops = {
		.elevator_merge_fn =		bfq_merge,
		.elevator_merged_fn =		bfq_merged_request,
		.elevator_merge_req_fn =	bfq_merged_requests,
		.elevator_allow_merge_fn =	bfq_allow_merge,
		.elevator_dispatch_fn =		bfq_dispatch_requests,
		.elevator_add_req_fn =		bfq_insert_request,
		.elevator_activate_req_fn =	bfq_activate_request,
		.elevator_deactivate_req_fn =	bfq_deactivate_request,
		.elevator_completed_req_fn =	bfq_completed_request,
		.elevator_former_req_fn =	elv_rb_former_request,
		.elevator_latter_req_fn =	elv_rb_latter_request,
		.elevator_init_icq_fn =		bfq_init_icq,
		.elevator_exit_icq_fn =		bfq_exit_icq,
		.elevator_set_req_fn =		bfq_set_request,
		.elevator_put_req_fn =		bfq_put_request,
		.elevator_may_queue_fn =	bfq_may_queue,
		.elevator_init_fn =		bfq_init_queue,
		.elevator_exit_fn =		bfq_exit_queue,
	},
	.icq_size =		sizeof(struct bfq_io_cq),
	.icq_align =		__alignof__(struct bfq_io_cq),
	.elevator_attrs =	bfq_attrs,
	.elevator_name =	"bfq",
	.elevator_owner =	THIS_MODULE,
};

static int __init bfq_init(void)
{
	/*
	 * Can be 0 on HZ < 1000 setups.
	 */
	if (bfq_slice_idle == 0)
		bfq_slice_idle = 1;

	if (bfq_timeout_async == 0)
		bfq_timeout_async = 1;

	if (bfq_slab_setup())
		return -ENOMEM;

	/*
	 * Times to load large popular applications for the typical systems
	 * installed on the reference devices (see the comments before the
	 * definitions of the two arrays).
	 */
	T_slow[0] = msecs_to_jiffies(2600);
	T_slow[1] = msecs_to_jiffies(1000);
	T_fast[0] = msecs_to_jiffies(5500);
	T_fast[1] = msecs_to_jiffies(2000);

	/*
	 * Thresholds that determine the switch between speed classes (see
	 * the comments before the definition of the array).
	 */
	device_speed_thresh[0] = (R_fast[0] + R_slow[0]) / 2;
	device_speed_thresh[1] = (R_fast[1] + R_slow[1]) / 2;

	elv_register(&iosched_bfq);
	pr_info("BFQ I/O-scheduler: v7r8");

	return 0;
}

static void __exit bfq_exit(void)
{
	elv_unregister(&iosched_bfq);
	bfq_slab_kill();
}

module_init(bfq_init);
module_exit(bfq_exit);

MODULE_AUTHOR("Fabio Checconi, Paolo Valente");
MODULE_LICENSE("GPL");