--- linux-5.18/block/blk-mq.c	2022-05-23 04:52:31.000000000 +0900
+++ /tmp/kernel-build/linux-5.18/block/blk-mq.c	2026-03-09 10:46:44.884274068 +0900
@@ -43,19 +43,277 @@
 #include "blk-mq-sched.h"
 #include "blk-rq-qos.h"
 
+/* Parameters for selective I/O completion mechanism */
+unsigned int benefit_ratio = 10;
+/* Parameters dependent on system configuration */
+unsigned long ctx_time = 1487;
+unsigned long isr_time = 1076; // EHP
+
+struct blk_switch __percpu *irq_poll_switch = NULL;
+EXPORT_SYMBOL(irq_poll_switch);
+
 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
 
 static void blk_mq_poll_stats_start(struct request_queue *q);
 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
 
+#define CPU_COUNT 20
+
+#define _INT 0
+#define _CP 1
+#define _PAS 2
+#define _OL 3
+#define _EHP 4
+
+int io_log_started = 0;
+
+static char log_first_line[] = "cpu,bucket,ret_num,sleep_requested,sleep_actual,io_time,timestamp\n";
+static char log_first_line_verbose[] = "code,cpu,lba,sectors,bucket,update_req,sr_pnlt,sr_last,dur_cnt,dur_cnt_checked,adj,up,dn,dur,sleep_actual,poll_time,poll_ret,io_time,rq_cpu_num,rq_dur,rq_dur_cnt,dur_cnt,dur_cnt_checked,sr_pnlt,sr_last,update_req,timestamp\n";
+static struct file* log_files[CPU_COUNT];
+static loff_t log_positions[CPU_COUNT];
+
+void start_io_log(void) {
+	int i;
+	char filename[] = "/media/ramdisk/pas_log_";
+	printk("Online CPUS: %d\n", num_online_cpus());
+	for(i=0;i<num_online_cpus();i++) {
+		char buffer[500] = {0, };
+		char core_buffer[10] = {0, };
+		log_positions[i] = 0;
+
+		sprintf(core_buffer, "%d", i);
+		strcat(buffer, filename);
+		strcat(buffer, core_buffer);
+		strcat(buffer, ".csv");
+		log_files[i] = filp_open(buffer,  O_CREAT | O_TRUNC | O_WRONLY, 0666);
+		kernel_write(log_files[i], log_first_line, 
+			strlen(log_first_line), &log_positions[i]);
+	}
+	io_log_started = 1;
+	printk("I/O Logging Started");
+}
+EXPORT_SYMBOL(start_io_log);
+
+void start_io_log_verbose(void) {
+	int i;
+	char filename[] = "/media/ramdisk/pas_log_";
+	printk("Online CPUS: %d\n", num_online_cpus());
+	for(i=0;i<num_online_cpus();i++) {
+		char buffer[500] = {0, };
+		char core_buffer[10] = {0, };
+		log_positions[i] = 0;
+
+		sprintf(core_buffer, "%d", i);
+		strcat(buffer, filename);
+		strcat(buffer, core_buffer);
+		strcat(buffer, ".csv");
+		log_files[i] = filp_open(buffer,  O_CREAT | O_TRUNC | O_WRONLY, 0666);
+		kernel_write(log_files[i], log_first_line_verbose, 
+			strlen(log_first_line_verbose), &log_positions[i]);
+	}
+	io_log_started = 2;
+	printk("I/O Logging Started");
+}
+EXPORT_SYMBOL(start_io_log_verbose);
+
+void end_io_log(void) {
+	int i;
+	io_log_started = 0;
+	
+	for(i=0;i<num_online_cpus();i++) {
+		log_positions[i] = 0;
+		filp_close(log_files[i], NULL);
+	}
+	printk("I/O Logging Disabled\n");
+}
+EXPORT_SYMBOL(end_io_log);
+
+  /*static char log_first_line[] = "
+  62 code,cpu,lba,sectors,bucket,
+  63 update_req,sr_pnlt,sr_last,
+  64 dur_cnt,dur_cnt_checked,adj,up,dn,dur,
+  65 sleep_actual,poll_time,poll_ret,io_time,
+  66 rq_cpu_num,rq_dur,rq_dur_cnt,
+  67 dur_cnt,dur_cnt_checked,
+  68 sr_pnlt,sr_last,update_req,
+  69 timestamp\n";
+  70 */
+
+void write_io_log(int cpu_num, int bucket, int ret_num, 
+	u64 sleep_requested, u64 sleep_actual, u64 io_time, u64 timestamp) {
+	char write_buffer[500] = {0, };
+	char cpu_num_buffer[5] = {0, };
+	char bucket_buffer[5] = {0, };
+	char ret_num_buffer[5] = {0, };
+	char sleep_requested_buffer[30] = {0, };
+	char sleep_actual_buffer[30] = {0, };
+	char io_time_buffer[30] = {0, };
+	char timestamp_buffer[30] = {0, };
+
+	sprintf(cpu_num_buffer, "%d,", cpu_num);
+	sprintf(bucket_buffer, "%d,", bucket);
+	sprintf(ret_num_buffer, "%d,", ret_num);
+	sprintf(sleep_requested_buffer, "%llu,", (unsigned long long)sleep_requested);
+	sprintf(sleep_actual_buffer, "%llu,", (unsigned long long)sleep_actual);
+	sprintf(io_time_buffer, "%llu,", (unsigned long long)io_time);
+	sprintf(timestamp_buffer, "%llu\n", (unsigned long long)timestamp);
+
+	strcat(write_buffer, cpu_num_buffer);
+	strcat(write_buffer, bucket_buffer);
+	strcat(write_buffer, ret_num_buffer);
+	strcat(write_buffer, sleep_requested_buffer);
+	strcat(write_buffer, sleep_actual_buffer);
+	strcat(write_buffer, io_time_buffer);
+	strcat(write_buffer, timestamp_buffer);
+	kernel_write(log_files[cpu_num], write_buffer, strlen(write_buffer),
+		&log_positions[cpu_num]);	
+}
+
+void write_io_log_verbose(char* code, int cpu_num, sector_t lba, unsigned int sectors, int bucket, 
+	u8 update_req_r, u8 sr_pnlt_r, u8 sr_last_r,
+	u8 dur_cnt_s, u8 dur_cnt_checked_r,
+	u64 adj, u64 up, u64 dn, u64 dur, 
+	u64 sleep_actual, u64 poll_time, int poll_ret, u64 io_time, 
+	int rq_cpu_num, u64 rq_dur, u8 rq_dur_cnt,
+	u8 dur_cnt_r, u8 dur_cnt_checked_s,
+	u8 sr_pnlt_s, u8 sr_last_s, u8 update_req_s, 
+	u64 timestamp) {
+	char write_buffer[500] = {0, };
+	char code_buffer[10] = {0, };
+	char cpu_num_buffer[5] = {0, };
+	char lba_buffer[20] = {0, };
+	char sectors_buffer[10] = {0, };
+	char bucket_buffer[5] = {0, };
+	char update_req_r_buffer[5] = {0, };
+	char sr_pnlt_r_buffer[5] = {0, };
+	char sr_last_r_rbuffer[5] = {0, };
+	char dur_cnt_s_buffer[5] = {0, };
+	char dur_cnt_checked_r_buffer[5] = {0, };
+	char adj_buffer[20] = {0, };
+	char up_buffer[20] = {0, };
+	char dn_buffer[20] = {0, };
+	char dur_buffer[20] = {0, };
+	char sleep_actual_buffer[20] = {0, };
+	char poll_time_buffer[20] = {0, };
+	char poll_ret_buffer[5] = {0, };
+	char io_time_buffer[30] = {0, };
+	char rq_cpu_num_buffer[5] = {0, };
+	char rq_dur_buffer[20] = {0, };
+	char rq_dur_cnt_buffer[5] = {0, };
+	char dur_cnt_r_buffer[5] = {0, };
+	char dur_cnt_checked_s_buffer[5] = {0, };
+	char sr_pnlt_s_buffer[5] = {0, };
+	char sr_last_s_buffer[5] = {0, };
+	char update_req_s_buffer[5] = {0, };
+	char timestamp_buffer[30] = {0, };
+
+	sprintf(code_buffer, "%s,", code);
+	sprintf(cpu_num_buffer, "%d,", cpu_num);
+	sprintf(lba_buffer, "%llu,", (unsigned long long)lba);
+	sprintf(sectors_buffer, "%u,", sectors);
+	sprintf(bucket_buffer, "%d,", bucket);
+	sprintf(update_req_r_buffer, "%u,", (unsigned int)update_req_r);
+	sprintf(sr_pnlt_r_buffer, "%u,", (unsigned int)sr_pnlt_r);
+	sprintf(sr_last_r_rbuffer, "%u,", (unsigned int)sr_last_r);
+	sprintf(dur_cnt_s_buffer, "%u,", (unsigned int)dur_cnt_s);
+	sprintf(dur_cnt_checked_r_buffer, "%u,", (unsigned int)dur_cnt_checked_r);
+	sprintf(adj_buffer, "%llu,", (unsigned long long)adj);
+	sprintf(up_buffer, "%llu,", (unsigned long long)up);
+	sprintf(dn_buffer, "%llu,", (unsigned long long)dn);
+	sprintf(dur_buffer, "%llu,", (unsigned long long)dur);
+	sprintf(sleep_actual_buffer, "%llu,", (unsigned long long)sleep_actual);
+	sprintf(poll_time_buffer, "%llu,", (unsigned long long)poll_time);
+	sprintf(poll_ret_buffer, "%d,", poll_ret);
+	sprintf(io_time_buffer, "%llu,", (unsigned long long)io_time);
+	sprintf(rq_cpu_num_buffer, "%d,", rq_cpu_num);
+	sprintf(rq_dur_buffer, "%llu,", (unsigned long long)rq_dur);
+	sprintf(rq_dur_cnt_buffer, "%u,", (unsigned int)rq_dur_cnt);
+	sprintf(dur_cnt_r_buffer, "%u,", (unsigned int)dur_cnt_r);
+	sprintf(dur_cnt_checked_s_buffer, "%u,", (unsigned int)dur_cnt_checked_s);
+	sprintf(sr_pnlt_s_buffer, "%u,", (unsigned int)sr_pnlt_s);
+	sprintf(sr_last_s_buffer, "%u,", (unsigned int)sr_last_s);
+	sprintf(update_req_s_buffer, "%u,", (unsigned int)update_req_s);
+	sprintf(timestamp_buffer, "%llu\n", (unsigned long long)timestamp);
+
+	strcat(write_buffer, code_buffer);
+	strcat(write_buffer, cpu_num_buffer);
+	strcat(write_buffer, lba_buffer);
+	strcat(write_buffer, sectors_buffer);
+	strcat(write_buffer, bucket_buffer);
+	strcat(write_buffer, update_req_r_buffer);
+	strcat(write_buffer, sr_pnlt_r_buffer);
+	strcat(write_buffer, sr_last_r_rbuffer);
+	strcat(write_buffer, dur_cnt_s_buffer);
+	strcat(write_buffer, dur_cnt_checked_r_buffer);
+	strcat(write_buffer, adj_buffer);
+	strcat(write_buffer, up_buffer);
+	strcat(write_buffer, dn_buffer);
+	strcat(write_buffer, dur_buffer);
+	strcat(write_buffer, sleep_actual_buffer);
+	strcat(write_buffer, poll_time_buffer);
+	strcat(write_buffer, poll_ret_buffer);
+	strcat(write_buffer, io_time_buffer);
+	strcat(write_buffer, rq_cpu_num_buffer);
+	strcat(write_buffer, rq_dur_buffer);
+	strcat(write_buffer, rq_dur_cnt_buffer);
+	strcat(write_buffer, dur_cnt_r_buffer);
+	strcat(write_buffer, dur_cnt_checked_s_buffer);
+	strcat(write_buffer, sr_pnlt_s_buffer);
+	strcat(write_buffer, sr_last_s_buffer);
+	strcat(write_buffer, update_req_s_buffer);
+	strcat(write_buffer, timestamp_buffer);
+
+	kernel_write(log_files[cpu_num], write_buffer, strlen(write_buffer), 
+		&log_positions[cpu_num]);
+}
+
+static void init_pas_stat(struct blk_rq_pas_stat *stat,  
+	u32 dur, long long adj, long long up, long long dn)
+{
+	stat->dur = dur;
+	stat->adj = adj;
+	stat->up = up;
+	stat->dn = dn;
+	stat->sr_pnlt = 0;
+	stat->sr_last = 1;
+	stat->dur_cnt = 1;
+	stat->dur_cnt_checked = 0;
+	stat->update_req = 0;
+	/*printk("init_pas_stat()(test_cnt: %llu)", stat->test_cnt);
+	printk("init_pas_stat: %p, dur: %llu, adj: %llu, up: %llu, dn: %llu, sr_pnlt: %d, sr_last: %d\n",
+		stat, stat->dur, stat->adj, stat->up, stat->dn, 
+		stat->sr_pnlt, stat->sr_last);
+	printk("init_pas_stat: %p, dur_cnt: %d, dur_cnt_checked: %d, update_req: %d\n", 
+		stat, stat->dur_cnt, stat->dur_cnt_checked, 
+		stat->update_req);*/
+}
+
+static void __maybe_unused restart_pas_stat(struct blk_rq_pas_stat *stat)
+{
+	stat->sr_pnlt = 0;
+	stat->sr_last = 1;
+	stat->dur_cnt = 1;
+	stat->dur_cnt_checked = 0;
+	stat->update_req = 0;
+}
+
 static int blk_mq_poll_stats_bkt(const struct request *rq)
 {
 	int ddir, sectors, bucket;
 
 	ddir = rq_data_dir(rq);
-	sectors = blk_rq_stats_sectors(rq);
+	sectors = blk_rq_sectors(rq);
+
+	// write = 1
+	// read = 0
 
 	bucket = ddir + 2 * ilog2(sectors);
+	//if (sectors >= 128)
+	//	bucket = ddir + 2 * 128;
+	//else
+	//	bucket = ddir + 2 * sectors;
+
+	//bucket = ddir +
 
 	if (bucket < 0)
 		return -1;
@@ -271,10 +529,8 @@
  * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
  * @q: request queue.
  *
- * Note: this function does not prevent that the struct request end_io()
- * callback function is invoked. Once this function is returned, we make
- * sure no dispatch can happen until the queue is unquiesced via
- * blk_mq_unquiesce_queue().
+ * This function recovers queue into the state before quiescing
+ * which is done by blk_mq_quiesce_queue.
  */
 void blk_mq_quiesce_queue(struct request_queue *q)
 {
@@ -290,6 +546,7 @@
  * This function recovers queue into the state before quiescing
  * which is done by blk_mq_quiesce_queue.
  */
+
 void blk_mq_unquiesce_queue(struct request_queue *q)
 {
 	unsigned long flags;
@@ -351,6 +608,7 @@
 	rq->cmd_flags = data->cmd_flags;
 
 	if (data->flags & BLK_MQ_REQ_PM)
+	
 		data->rq_flags |= RQF_PM;
 	if (blk_queue_io_stat(q))
 		data->rq_flags |= RQF_IO_STAT;
@@ -1216,6 +1474,7 @@
 		bio_poll(rq->bio, NULL, 0);
 		cond_resched();
 	} while (!completion_done(wait));
+		
 }
 
 /**
@@ -2845,7 +3104,8 @@
 		return;
 	}
 
-	if (plug)
+	if (plug &&
+	    (rq->bio->bi_opf & (REQ_POLLED | REQ_NOWAIT)) != REQ_POLLED)
 		blk_add_rq_to_plug(plug, rq);
 	else if ((rq->rq_flags & RQF_ELV) ||
 		 (rq->mq_hctx->dispatch_busy &&
@@ -4021,9 +4281,11 @@
 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
 		struct request_queue *q)
 {
+	int cpu;
+	int bucket_idx;
+	struct blk_rq_pas_stat* stat;
 	WARN_ON_ONCE(blk_queue_has_srcu(q) !=
 			!!(set->flags & BLK_MQ_F_BLOCKING));
-
 	/* mark the queue as mq asap */
 	q->mq_ops = set->ops;
 
@@ -4067,6 +4329,36 @@
 	 */
 	q->poll_nsec = BLK_MQ_POLL_CLASSIC;
 
+	/*
+	* PAS init
+	*/
+	q->max_no_lock = 100;
+	q->poll_threshold = 0;
+	q->div     = 1000000;
+	q->d_init  = 100; // unit: ns
+	q->up_init = 0.01 * 1000000; // 10000;
+	q->dn_init = 0.1  * 1000000; // 100000;
+	q->heat_up = 0.05 * 1000000; // 50000;
+	q->cool_dn = 0.1  * 1000000; // 100000;
+	q->min_dn  = 0.01 * 1000000; // 10000;
+	q->max_dn  = 0.1  * 1000000; // 100000;
+	q->updn_ratio = 10;
+
+/*	q->cnt_rel_hybrid_poll = 0;
+	q->cnt_rel_fops = 0;
+	q->cnt_rel_comp_before_sleep = 0;
+	q->cnt_lock_d_c_separate = 0;
+*/
+	q->pas_stat = __alloc_percpu(BLK_MQ_POLL_STATS_BKTS * 
+		sizeof(struct blk_rq_pas_stat),__alignof__(struct blk_rq_pas_stat));
+
+	for_each_possible_cpu(cpu) {
+		stat = per_cpu_ptr(q->pas_stat, cpu);
+        	for(bucket_idx=0;bucket_idx<BLK_MQ_POLL_STATS_BKTS;bucket_idx++)
+        		init_pas_stat(&stat[bucket_idx], q->d_init,
+				q->div, q->up_init, q->dn_init);
+	}
+
 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
 	blk_mq_add_queue_tag_set(set, q);
 	blk_mq_map_swqueue(q);
@@ -4519,8 +4811,7 @@
 	}
 
 	prev_nr_hw_queues = set->nr_hw_queues;
-	if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
-	    0)
+	if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) < 0)
 		goto reregister;
 
 	set->nr_hw_queues = nr_hw_queues;
@@ -4584,6 +4875,9 @@
 	if (!q->poll_stat || blk_stat_is_active(q->poll_cb))
 		return;
 
+	if(q->ehp_enabled)
+		blk_stat_activate_msecs(q->poll_cb, 10); // EHP
+	else
 	blk_stat_activate_msecs(q->poll_cb, 100);
 }
 
@@ -4591,8 +4885,32 @@
 {
 	struct request_queue *q = cb->data;
 	int bucket;
-
+	struct blk_switch *sc;
 	for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
+		int cpu_num = smp_processor_id();
+		struct blk_rq_stat *percpu_stat = per_cpu_ptr(cb->percpu_stat, 
+				cpu_num);
+		sc = per_cpu_ptr(irq_poll_switch, cpu_num);
+		if (q->ehp_enabled && percpu_stat[bucket].nr_samples) {
+		//if (percpu_stat[bucket].nr_samples) {
+			//printk("bucket in blk %d\n", bucket);
+			
+			if(sc->mode != _EHP)
+				sc->mode = _EHP;
+			if ((ctx_time * 2 + isr_time) * benefit_ratio > 
+					percpu_stat[bucket].min){
+				//sc->mode = _EHP;
+				sc->ehpmode[bucket] = _EHP;
+				//printk("here");
+			}
+			else{
+				//sc->mode = _INT;
+				sc->ehpmode[bucket] = _INT;
+				//percpu_stat[bucket].busy_state = 1; 
+			}
+			percpu_stat[bucket].busy_state = 0; 
+		} //EHP
+
 		if (cb->stat[bucket].nr_samples)
 			q->poll_stat[bucket] = cb->stat[bucket];
 	}
@@ -4630,6 +4948,311 @@
 	return ret;
 }
 
+static unsigned long blk_mq_poll_pas_nsecs(struct request_queue *q,
+					struct request *rq, int cpu_num)
+{
+	unsigned long ret = 0;
+	struct blk_rq_pas_stat *stat;
+	int bucket;
+	int cur_case;
+	//int cpu_num;
+	struct blk_switch *sc;
+	int average_qd;
+	unsigned long flags;
+	bucket = blk_mq_poll_stats_bkt(rq);
+
+	if (bucket < 0)
+		return ret;
+
+	//cpu_num = blk_mq_rq_cpu(rq);
+	stat = per_cpu_ptr(q->pas_stat, cpu_num);
+
+	sc = per_cpu_ptr(irq_poll_switch, cpu_num);
+
+	if(sc->enabled) {
+		if(sc->mode != _CP) {
+			spin_lock_irqsave(&sc->qd_lock, flags);
+			sc->qd++;
+			sc->qd_sum += sc->qd;
+			spin_unlock_irqrestore(&sc->qd_lock, flags);
+		}
+//		if(sc->N_PAS == 1){
+//			stat[bucket].update_req = 1;
+//			sc->N_PAS = 0;
+//		}
+	}
+
+
+
+
+//	if(sc->ioctr < 10000) 
+//		printk("REQ[%d] CPU[%2d] LOCK[%d] tf[%2d] qd[%d] qdsum[%2d] MODE[%d] cnt: [%3d|%3d|%3d|%3d] tot: [%7llu|%7llu|%7llu|%7llu]",
+//			stat[bucket].update_req, cpu_num, sc->N_POLL, sc->tf, sc->qd, sc->qd_sum, sc->mode,   
+//			sc->cp_cnt, sc->pas_cnt, sc->ol_cnt, sc->int_cnt,
+//			sc->cp_tot, sc->pas_tot, sc->ol_tot, sc->int_tot);
+
+
+
+
+
+
+
+
+//	if(sc->enabled && sc->ioctr >= 100000) 
+//		sc->ioctr = 0;
+
+	//if(sc->enabled && sc->ioctr >= 100000) {
+	//	sc->ioctr = 0;
+	//	printk("REQ[%d] CPU[%2d] qdsum[%2d] MODE[%d] cnt: [%3d|%3d|%3d|%3d] tot: [%7llu|%7llu|%7llu|%7llu]",
+	//		stat[bucket].update_req, cpu_num, sc->qd_sum, sc->mode,   
+	//		sc->cp_cnt, sc->pas_cnt, sc->ol_cnt, sc->int_cnt,
+	//		sc->cp_tot, sc->pas_tot, sc->ol_tot, sc->int_tot);
+	//}
+
+/*
+	if(stat[bucket].update_req == 0) { 
+		rq->cpu_num = cpu_num;
+		rq->dur_cnt = stat[bucket].dur_cnt;
+		if(sc->enabled && (sc->mode == _CP)) {  // CP
+			if(sc->cp_cnt >= 1000) { //sc->N_POLL) {
+				sc->mode = _PAS; //CP->PAS 
+				sc->pas_cnt = 0;
+				sc->tf = 0;
+				sc->qd_sum = 0;
+				stat[bucket].update_req = 1;
+				if(sc->ioctr <= 1000) 
+					printk("CPU%d %7d CP1->PAS", cpu_num, sc->ioctr);
+			}
+			return ret;
+		}
+		else {
+			return stat[bucket].dur;
+		}
+	}
+*/
+
+/*
+	if(sc->enabled && (sc->mode == _CP)) {  // CP
+		if(sc->cp_cnt >= 1000) { //sc->N_POLL) {
+			stat[bucket].update_req = 1;
+			sc->mode = _PAS; //CP->PAS 
+			sc->pas_cnt = 0;
+			sc->tf = 0;
+//			sc->qd = 0;
+			sc->qd_sum = 0;
+			restart_pas_stat(stat); // replace the below line.
+			if(sc->ioctr <= 1000) 
+				printk("CPU%d %7d CP1->PAS", cpu_num, sc->ioctr);
+		}
+		return ret;
+	}
+*/
+
+
+	if(sc->enabled && !sc->N_POLL) { // if lock is available. mode should be _PAS or _OL.
+		sc->N_POLL = 1; // acquire lock
+		if(sc->mode == _CP && sc->cp_cnt >= 1000) { //sc->N_POLL) {
+			stat[bucket].update_req = 1;
+			sc->mode = _PAS; /* CP->PAS */
+			sc->pas_cnt = 0;
+			sc->tf = 0;
+//			sc->qd = 0;
+			sc->qd_sum = 0;
+//			if(sc->ioctr <= 10000) 
+//				printk("CPU%d %7d CP2->PAS", cpu_num, sc->ioctr);
+		}
+		else if(sc->mode == _PAS && (sc->pas_cnt >= 100)) { // PAS
+			average_qd = sc->qd_sum * 10 / sc->pas_cnt; // NPAS=100
+			//average_qd = sc->qd_sum * 10 / 100; // NPAS=100
+			if(sc->param4 >= 1 && average_qd == 10) { // QD check first (deterministic). 10x QD is used. param4 = 0: PAS->CP disabled. param4 = 1: PAS->CP enabled.
+				sc->mode = _CP; // PAS->CP when average QD <= 1
+				sc->cp_cnt = 0;
+//				sc->qd = 0;
+				sc->qd_sum = 0; // delete later.
+				sc->tf = 0; // delete later.
+			} else if(sc->tf > sc->param1) { // tf check second (environment-sensitive)
+				sc->mode = _OL; // PAS->OL when timer is not responding
+				sc->ol_cnt = 0;
+//				sc->qd = 0;
+				sc->qd_sum = 0;
+				sc->tf = 0;
+			} else { // stay PAS
+				sc->pas_cnt = 0;
+				//sc->qd = 0;
+				sc->qd_sum = 0;
+				sc->tf = 0; // delete later.
+			}
+		} 
+		else if(sc->mode == _OL && (sc->ol_cnt >= 100)) { /* OL */
+			average_qd = sc->qd_sum * 10 / sc->ol_cnt; // NPAS=100 
+			//average_qd = sc->qd_sum * 10 / 100; // NPAS=100 
+//			if(sc->ioctr <= 10000)
+//				printk("OL eval: qdsum: %d olcnt: %d aveqd: %d", sc->qd_sum, sc->ol_cnt, average_qd);
+			if(average_qd <= sc->param2) {
+//				if(sc->ioctr <= 10000) 
+//					printk("CPU%d %7d tf:%d aveqd: %d OL->PAS", cpu_num, sc->ioctr, sc->tf, average_qd );
+				sc->mode = _PAS; // OL->PAS when average QD < 1
+				sc->pas_cnt = 0;
+//				sc->qd = 0;
+				sc->qd_sum = 0;
+				sc->tf = 0;
+			}
+			else if(average_qd > sc->param3) {
+//				if(1 || sc->ioctr <= 10000) {
+//					printk("OL eval: qdsum: %d olcnt: %d aveqd: %d", sc->qd_sum, sc->ol_cnt, average_qd);
+//					printk("CPU%d %7d tf:%d aveqd: %d OL->INT", cpu_num, sc->ioctr, sc->tf, average_qd );
+//				}
+				sc->mode = _INT; /* OL -> INT when average QD >= 2 */
+				sc->int_cnt = 0;
+//				sc->qd = 0;
+				sc->qd_sum = 0; // delete later
+				sc->tf = 0; // delete later
+			}
+			else { // stay OL
+//				if(sc->ioctr <= 10000) 
+//					printk("CPU%d %7d tf:%d aveqd: %d OL->OL", cpu_num, sc->ioctr, sc->tf, average_qd );
+				sc->ol_cnt = 0;
+				//sc->qd = 0;
+				sc->qd_sum = 0;
+				sc->tf = 0;
+			}
+		}
+		sc->N_POLL = 0; // release lock.
+	}
+
+	if(sc->enabled && sc->mode == _CP)
+		return ret;
+
+	if(stat[bucket].update_req == 0) { 
+		rq->cpu_num = cpu_num;
+		rq->dur_cnt = stat[bucket].dur_cnt;
+		return stat[bucket].dur;
+	}
+
+	stat[bucket].update_req = 0;
+
+	/* Update sleep duration. */
+	cur_case = stat[bucket].sr_pnlt * 2 + stat[bucket].sr_last;
+	switch(cur_case) {
+		case 0: /* [sr_pnlt, sr_last] = [overslept, overslept] */
+			stat[bucket].adj -= stat[bucket].dn;
+			break;
+		case 1: /* [sr_pnlt, sr_last] = [overslept, underslept] */
+			stat[bucket].adj = 1 * q->div + stat[bucket].up;
+			break;
+		case 2: /* [sr_pnlt, sr_last] = [underslept, overslept] */
+			stat[bucket].adj = 1 * q->div - stat[bucket].dn;
+			break;
+		case 3: /* [sr_pnlt, sr_last] = [underslept, underslept] */
+			stat[bucket].adj += stat[bucket].up;
+			break;
+		default:
+			stat[bucket].adj = 1 * q->div;
+	}
+	if (stat[bucket].adj <= 0) /* reset adj to 1 if it is zero or less */
+		stat[bucket].adj = 1 * q->div; 
+	stat[bucket].dur = 
+		stat[bucket].dur * stat[bucket].adj / q->div;
+	if (stat[bucket].dur < q->d_init) {
+		stat[bucket].dur = q->d_init;
+		if(sc->enabled) {
+			sc->tf++;
+		}
+	} 
+	stat[bucket].dur_cnt++;
+
+ 	/* Dynamically adjust up and dn. */
+	if(q->pas_adaptive_enabled) {
+		/* Heat up when sr_pnlt is equal to sr_last */
+		if (cur_case == 0 || cur_case == 3) {
+			if(q->pas_adaptive_enabled == 1) { // DPAS1: both up and dn
+				stat[bucket].dn = stat[bucket].dn * 
+					(1 * q->div + q->heat_up) / q->div; 
+				if (stat[bucket].dn > q->max_dn)
+					stat[bucket].dn = q->max_dn;
+				stat[bucket].up = stat[bucket].dn / q->updn_ratio;
+			} else if(q->pas_adaptive_enabled == 2) { // DPAS2: up only
+				stat[bucket].up = stat[bucket].up * 
+					(1 * q->div + q->heat_up) / q->div;
+				if (stat[bucket].up > q->div / 10) // max_up: 0.1
+					stat[bucket].up = q->div / 10;
+			}
+		} else { 
+		/* Cool down when sr_pnlt is different from sr_last */
+			stat[bucket].up = stat[bucket].up * 
+				( 1 * q->div - q->cool_dn) / q->div;
+			if(q->pas_adaptive_enabled == 1) { // DPAS1
+				if(stat[bucket].up < q->min_dn / q->updn_ratio)
+					stat[bucket].up = q->min_dn / q->updn_ratio;
+				stat[bucket].dn = stat[bucket].up * q->updn_ratio;
+			} else if(q->pas_adaptive_enabled == 2) { // DPAS2
+				if (stat[bucket].up < q->div / 10000){ // min_up: 0.0001
+					stat[bucket].up = q->div / 10000;
+				}
+			}
+		}
+	}
+		
+	ret = stat[bucket].dur;
+	rq->cpu_num = cpu_num;
+	rq->dur_cnt = stat[bucket].dur_cnt;
+/*	if(stat[bucket].test_cnt < 100) {
+		printk("UPDATE CPU%d 1exit blk_mq_poll_pas_nsecs(test_cnt: %llu)", cpu_num, stat[bucket].test_cnt);
+		printk("1rq->dur_cnt: %d, rq->dur: %llu\n", rq->dur_cnt, rq->dur);
+		printk("1exit stat: %p, dur: %llu, adj: %llu, up: %llu, dn: %llu, sr_pnlt: %d, sr_last: %d\n",
+			&(stat[bucket]), stat[bucket].dur, stat[bucket].adj, stat[bucket].up, stat[bucket].dn, 
+			stat[bucket].sr_pnlt, stat[bucket].sr_last);
+		printk("1exit stat: %p, dur_cnt: %d, dur_cnt_checked: %d, update_req: %d\n", 
+			&(stat[bucket]), stat[bucket].dur_cnt, stat[bucket].dur_cnt_checked, 
+			stat[bucket].update_req);
+	}
+*/	
+	return ret;
+}
+
+static unsigned long blk_mq_poll_ehp_nsecs(struct request_queue *q, struct request *rq)
+{
+	unsigned long ret = 0;
+	int bucket;
+	struct blk_stat_callback *cb;
+	struct blk_rq_stat *percpu_stat;
+
+	/*
+	* If stats collection isn't on, don't sleep but turn it on for
+	* future users
+	*/
+	if (!blk_poll_stats_enable(q))
+		return 0;
+
+    	/*
+     	* As an optimistic guess, use half of the mean service time
+     	* for this type of request. We can (and should) make this smarter.
+     	* For instance, if the completion latencies are tight, we can
+     	* get closer than just half the mean. This is especially
+     	* important on devices where the completion latencies are longer
+     	* than ~10 usec. We do use the stats for the relevant IO size
+     	* if available which does lead to better estimates.
+     	*/
+    	bucket = blk_mq_poll_stats_bkt(rq);
+    	if (bucket < 0)
+        	return ret;
+
+    	cb = q->poll_cb;
+    	//percpu_stat = per_cpu_ptr(cb->percpu_stat, raw_smp_processor_id());
+    	percpu_stat = per_cpu_ptr(cb->percpu_stat, smp_processor_id());
+
+    	if (percpu_stat[bucket].nr_samples) {
+    		if (percpu_stat[bucket].busy_state) {
+    			ret = 4294967295;
+       		} else if (percpu_stat[bucket].min > 0 &&
+        		(percpu_stat[bucket].min > (ctx_time * 2 + isr_time))) {
+            		ret = percpu_stat[bucket].min - ctx_time * 2 - isr_time;
+        	}
+    	}
+
+    	return ret;
+}
+
 static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
 {
 	struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, qc);
@@ -4638,6 +5261,22 @@
 	enum hrtimer_mode mode;
 	unsigned int nsecs;
 	ktime_t kt;
+	struct blk_switch *sc;
+	int cpu_num;
+
+	/* Variables for write_io_log_verbose() */
+    	int log_cpu_num;
+	sector_t log_lba;
+	u64 log_sectors;
+	struct blk_rq_pas_stat* stat;
+	int log_bucket;
+	u8 log_update_req_r, log_sr_pnlt_r, log_sr_last_r;
+	u8 log_dur_cnt_s, log_dur_cnt_checked_r;
+	u64 log_adj, log_up, log_dn, log_dur;
+	int log_cpu_num_after_sleep;
+        u64 log_sleep_start_time;
+	u64 log_io_start_time;
+	unsigned long flags;
 
 	/*
 	 * If a request has completed on queue that uses an I/O scheduler, we
@@ -4646,16 +5285,92 @@
 	if (!rq || (rq->rq_flags & RQF_MQ_POLL_SLEPT))
 		return false;
 
+	if(io_log_started == 2) { // Type 1 log: from I/O submission to sleep
+		log_io_start_time = rq->io_start_time_ns;
+    		log_cpu_num = blk_mq_rq_cpu(rq);
+		log_lba = blk_rq_pos(rq);
+		log_sectors = blk_rq_sectors(rq);
+		if(q->pas_enabled) stat = per_cpu_ptr(q->pas_stat, log_cpu_num);
+		log_bucket = blk_mq_poll_stats_bkt(rq);
+		if(q->pas_enabled) {
+			log_update_req_r = stat[log_bucket].update_req;
+			log_sr_pnlt_r = stat[log_bucket].sr_pnlt;
+			log_sr_last_r = stat[log_bucket].sr_last;
+		log_dur_cnt_s = 8;
+			log_dur_cnt_checked_r = stat[log_bucket].dur_cnt_checked;
+		}
+		else {
+			log_update_req_r = 8;
+			log_sr_pnlt_r = 8;
+			log_sr_last_r = 8;
+			log_dur_cnt_s = 8;
+			log_dur_cnt_checked_r = 8;
+		}
+		log_adj = 8;
+		log_up = 8;
+		log_dn = 8;
+		log_dur = 8;
+	}
+
 	/*
 	 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
 	 *
 	 *  0:	use half of prev avg
 	 * >0:	use this specific value
 	 */
-	if (q->poll_nsec > 0)
+	if (q->poll_nsec > 0) /* LHP with a fixed sleep duration */
 		nsecs = q->poll_nsec;
-	else
+	else if(q->ehp_enabled){ /* EHP */
+		nsecs = blk_mq_poll_ehp_nsecs(q, rq);
+		if (nsecs == 4294967295){
+			if(io_log_started == 2) {
+				log_dur = 0;
+				/* Before Sleep, Sleep Zero, EHp. */
+				write_io_log_verbose("BSSZEH", log_cpu_num, log_lba, log_sectors, log_bucket,
+					log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+					log_dur_cnt_s, log_dur_cnt_checked_r,
+					log_adj, log_up, log_dn, log_dur,
+					8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+					log_io_start_time);
+			}
+			//return false;
+		}
+	} else if(q->pas_enabled){ /* PAS */
+		cpu_num = blk_mq_rq_cpu(rq);
+		nsecs = blk_mq_poll_pas_nsecs(q, rq, cpu_num);
+		if(io_log_started == 2) {
+			log_adj = stat[log_bucket].adj;
+			log_up = stat[log_bucket].up;
+			log_dn = stat[log_bucket].dn;
+		}
+	}
+	else{ /* LHP with an adaptive sleep duration */
 		nsecs = blk_mq_poll_nsecs(q, rq);
+	}
+
+	if(io_log_started == 1) {
+		rq->dur = nsecs;
+	} else if(io_log_started == 2){ // Log for EHP, LHP and PAS
+		rq->dur = log_dur = nsecs;
+		if(q->pas_enabled)
+			log_dur_cnt_s = stat[log_bucket].dur_cnt;
+		if(!nsecs)
+			// Before Sleep, Sleep Zero, ALl methods.
+			write_io_log_verbose("BSSZAL", log_cpu_num, log_lba, log_sectors, log_bucket,
+				log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				log_dur_cnt_s, log_dur_cnt_checked_r,
+				log_adj, log_up, log_dn, log_dur,
+				8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+				log_io_start_time);
+		else
+			// Before Sleep, Sleep Success, ALl methods.
+			write_io_log_verbose("BSSSAL", log_cpu_num, log_lba, log_sectors, log_bucket,
+				log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				log_dur_cnt_s, log_dur_cnt_checked_r,
+				log_adj, log_up, log_dn, log_dur,
+				8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+				log_io_start_time);
+	}
 
 	if (!nsecs)
 		return false;
@@ -4668,21 +5383,84 @@
 	 */
 	kt = nsecs;
 
+	if(io_log_started == 1 || io_log_started == 2)
+		log_sleep_start_time = ktime_get_real_ns();
+
 	mode = HRTIMER_MODE_REL;
 	hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
 	hrtimer_set_expires(&hs.timer, kt);
 
 	do {
-		if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
+		if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) {
+			if(io_log_started == 2)
+				// Before Sleep, Sleep Cancelled, ALl methods.
+				write_io_log_verbose("BSSCAL", log_cpu_num, log_lba, log_sectors, log_bucket,
+					log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+					log_dur_cnt_s, log_dur_cnt_checked_r,
+					log_adj, log_up, log_dn, log_dur,
+					8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+					log_io_start_time);
 			break;
+		}
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		hrtimer_sleeper_start_expires(&hs, mode);
-		if (hs.task)
-			io_schedule();
+		io_schedule(); /* always yield CPU, even if timer already expired */
 		hrtimer_cancel(&hs.timer);
 		mode = HRTIMER_MODE_ABS;
 	} while (hs.task && !signal_pending(current));
 
+	if(q->pas_enabled && (irq_poll_switch != NULL)) {
+		sc = per_cpu_ptr(irq_poll_switch, cpu_num);
+		spin_lock_irqsave(&sc->qd_lock, flags);
+		sc->qd = sc->qd - 1;
+		spin_unlock_irqrestore(&sc->qd_lock, flags);
+/*		if(cpu_num == blk_mq_rq_cpu(rq)) {
+			sc = per_cpu_ptr(irq_poll_switch, cpu_num);
+			if(--(sc->qd) < 0) {
+				sc->qd = 0;
+				//printk("QD underrun.");
+			} 
+		}
+		else {
+			printk("CPU CHG: skip decreasing QD.");
+		}*/
+	}
+
+	if(io_log_started == 1) {
+		rq->log_real_sleep_time = ktime_get_real_ns() - log_sleep_start_time;
+	} else if(io_log_started == 2) {
+		if(log_lba != blk_rq_pos(rq)) {
+			// After Sleep, Already Reaped, ALl methods.
+			write_io_log_verbose("ASARAL", log_cpu_num, log_lba, log_sectors, log_bucket,
+				8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+				ktime_get_real_ns() - log_sleep_start_time, 0, 0, 
+				ktime_get_ns() - log_io_start_time,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				ktime_get_ns());
+		} else { 
+			rq->log_real_sleep_time = ktime_get_real_ns() - log_sleep_start_time;
+			log_cpu_num_after_sleep = blk_mq_rq_cpu(rq);
+			if(log_cpu_num != log_cpu_num_after_sleep) {
+			// Cpu Change ALl methods: Before & After
+			write_io_log_verbose("CCALBA", log_cpu_num, log_lba, log_sectors, log_bucket,
+				0, 0, 0, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				0, 0, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				0, 0, 0, 0, //log_adj, log_up, log_dn, log_dur,
+				8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+				ktime_get_ns());
+			// Cpu Change ALl methods: After & Before
+			write_io_log_verbose("CCALAB", log_cpu_num_after_sleep, log_lba, log_sectors, log_bucket,
+				0, 0, 0, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				0, 0, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				0, 0, 0, 0, //log_adj, log_up, log_dn, log_dur,
+				8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 
+				ktime_get_ns());
+			}
+		}
+	}
+
 	__set_current_state(TASK_RUNNING);
 	destroy_hrtimer_on_stack(&hs.timer);
 
@@ -4696,6 +5474,61 @@
 	return true;
 }
 
+
+/*
+static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
+			       struct io_comp_batch *iob, unsigned int flags)
+{
+	struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
+	long state = get_current_state();
+	int ret;
+
+	struct request *rq = blk_qc_to_rq(hctx, cookie);
+	struct blk_rq_pas_stat* stat = 0;
+	int cpu_num; 
+	int bucket;
+	int poll_count = 0;
+
+	do {
+		ret = q->mq_ops->poll(hctx, iob);
+		if (ret > 0) {
+			if(q->pas_enabled && rq) {
+				cpu_num = blk_mq_rq_cpu(rq);
+				bucket = blk_mq_poll_stats_bkt(rq);
+				if(rq->cpu_num == cpu_num)
+					stat = per_cpu_ptr(q->pas_stat, rq->cpu_num);
+				if(rq->cpu_num == cpu_num && 
+					rq->dur_cnt == stat[bucket].dur_cnt &&
+					stat[bucket].dur_cnt != stat[bucket].dur_cnt_checked) {
+					stat[bucket].dur_cnt_checked = stat[bucket].dur_cnt;
+					stat[bucket].sr_pnlt = stat[bucket].sr_last;
+					if(!poll_count)
+						stat[bucket].sr_last = 0;
+					else
+						stat[bucket].sr_last = 1;					
+					stat[bucket].update_req = 1;
+				}
+			}
+			__set_current_state(TASK_RUNNING);
+			return ret;
+		}
+
+		if (signal_pending_state(state, current))
+			__set_current_state(TASK_RUNNING);
+		if (task_is_running(current)){
+			return 1;
+		}
+
+		if (ret < 0 || (flags & BLK_POLL_ONESHOT))
+			break;
+                poll_count++;
+		cpu_relax();
+	} while (!need_resched());
+
+	return 0;
+}
+*/
+
 static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
 			       struct io_comp_batch *iob, unsigned int flags)
 {
@@ -4703,23 +5536,193 @@
 	long state = get_current_state();
 	int ret;
 
+	struct request *rq = blk_qc_to_rq(hctx, cookie);
+	struct blk_rq_pas_stat* stat = 0;
+	int cpu_num; 
+	int bucket;
+	int poll_count = 0;
+	//struct blk_switch *sc;
+
+	// Variables for write_io_log_verbose() 
+	u64 poll_start_time;
+    	int log_cpu_num = 0;
+	sector_t log_lba = 0;
+	u64 log_sectors = 0;
+	int log_bucket = 0;
+        u64 log_sleep_actual_time = 0; u64 log_poll_time = 8; 
+	int log_poll_ret = 8; u64 log_io_time = 8;
+	int log_rq_cpu_num = 8; u64 log_rq_dur = 8; u8 log_rq_dur_cnt = 8;
+	u8 log_dur_cnt_r = 8; u8 log_dur_cnt_checked_s = 8;
+	u8 log_sr_pnlt_s = 8; u8 log_sr_last_s = 8; u8 log_update_req_s = 8;
+
+	if(rq && (io_log_started == 2)) { // Type 2 log: from waking up to I/O completion
+		log_cpu_num = blk_mq_rq_cpu(rq);
+		log_lba = blk_rq_pos(rq);
+		log_sectors = blk_rq_sectors(rq);
+		log_bucket = blk_mq_poll_stats_bkt(rq);
+		log_sleep_actual_time = rq->log_real_sleep_time;
+		if(q->pas_enabled) {
+			log_rq_cpu_num = rq->cpu_num;
+			log_rq_dur = rq->dur;
+			log_rq_dur_cnt = rq->dur_cnt;
+		}
+		//io_start_time: stored at rq->io_start_time_ns.
+		poll_start_time = ktime_get_real_ns();
+	}
+
 	do {
 		ret = q->mq_ops->poll(hctx, iob);
 		if (ret > 0) {
+			if(io_log_started == 1 && rq)
+				write_io_log(blk_mq_rq_cpu(rq), blk_mq_poll_stats_bkt(rq), ret,
+					rq->dur, rq->log_real_sleep_time, 
+					ktime_get_ns() - rq->io_start_time_ns, ktime_get_ns());
+			else if(io_log_started == 2) { // for all poll-based methods. 
+				log_poll_time = ktime_get_real_ns() - poll_start_time;
+				log_poll_ret = ret;
+				if(rq) 
+					log_io_time = ktime_get_ns() - rq->io_start_time_ns;
+			}
+			if(q->pas_enabled && rq) {
+				cpu_num = blk_mq_rq_cpu(rq);
+				bucket = blk_mq_poll_stats_bkt(rq);
+				if(rq->cpu_num == cpu_num)
+					stat = per_cpu_ptr(q->pas_stat, rq->cpu_num);
+				//else 
+				//	q->cnt_lock_d_c_separate++;
+				//if(irq_poll_switch != NULL) { /* update QD */
+				//	sc = per_cpu_ptr(irq_poll_switch, cpu_num);
+				//	sc->qd -= ret;
+				//	if(sc->qd < 0) sc->qd = 0;
+//				}
+
+				if(rq->cpu_num == cpu_num && 
+					rq->dur_cnt == stat[bucket].dur_cnt &&
+					stat[bucket].dur_cnt != stat[bucket].dur_cnt_checked) {
+					stat[bucket].dur_cnt_checked = stat[bucket].dur_cnt;
+					stat[bucket].sr_pnlt = stat[bucket].sr_last;
+					if(!poll_count)
+					//if(poll_count <= q->poll_threshold) // default threshold: 0 
+						stat[bucket].sr_last = 0;
+					else
+						stat[bucket].sr_last = 1;					
+					stat[bucket].update_req = 1;
+				}
+ 
+				if(io_log_started == 2 && stat) {
+					log_dur_cnt_r = stat[bucket].dur_cnt;
+					log_dur_cnt_checked_s = stat[bucket].dur_cnt_checked;
+					log_sr_pnlt_s = stat[bucket].sr_pnlt;
+					log_sr_last_s = stat[bucket].sr_last;
+					log_update_req_s = stat[bucket].update_req;
+				}
+			}
+
+			if (io_log_started == 2 && rq) { // When poll completes.
+				if (q->pas_enabled) {
+					if(log_update_req_s) // After Sleep, Poll Success, Pas Update sleep results.
+						write_io_log_verbose("ASPSPU", log_cpu_num, log_lba, log_sectors, log_bucket,
+							8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+							8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+							8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+							log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+							log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+							log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+							log_update_req_s, ktime_get_ns());
+					else // After Sleep, Poll Success, Pas Not update sleep results.
+						write_io_log_verbose("ASPSPN", log_cpu_num, log_lba, log_sectors, log_bucket,
+							8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+							8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+							8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+							log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+							log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+							log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+							log_update_req_s, ktime_get_ns());
+				} else if (q->poll_nsec == 0)	// LHP, EHP
+					write_io_log_verbose("ASPSLE", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Poll Success, Lhp, Ehp.
+						8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+						8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+						8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+						log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+						8, 8, 8, // log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+						8, 8, 8, 8, // log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+						8, ktime_get_ns());
+				else				// CP
+					write_io_log_verbose("ASPSCP", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Poll Success, Classic Polling.
+						8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+						8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+						8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+						8, log_poll_time, log_poll_ret, log_io_time,
+						8, 8, 8, // log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+						8, 8, 8, 8, // log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+						8, ktime_get_ns());
+			} else if(io_log_started == 2 && !rq){
+				write_io_log_verbose("ASPSRN", 0, log_lba, log_sectors, log_bucket, // After Sleep, Poll Success, Rq Null.
+					8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+					8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+					8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+					8, log_poll_time, log_poll_ret, log_io_time,
+					8, 8, 8, // log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+					8, 8, 8, 8, // log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+					8, ktime_get_ns());
+			}
 			__set_current_state(TASK_RUNNING);
 			return ret;
 		}
 
 		if (signal_pending_state(state, current))
 			__set_current_state(TASK_RUNNING);
-		if (task_is_running(current))
+		if (task_is_running(current)){
+			if(rq && io_log_started == 2){
+				log_poll_time = ktime_get_real_ns() - poll_start_time;
+				write_io_log_verbose("ASSPAL", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Signal Pending, ALl methods.
+					8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+					8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+					8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+					log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+					log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+					log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+					log_update_req_s, ktime_get_ns());
+			}
 			return 1;
+		}
 
 		if (ret < 0 || (flags & BLK_POLL_ONESHOT))
 			break;
+                poll_count++;
 		cpu_relax();
 	} while (!need_resched());
 
+	if (io_log_started == 2) {
+		log_poll_time = ktime_get_real_ns() - poll_start_time;
+		if (q->pas_enabled)  		// PAS
+			write_io_log_verbose("ASNSPA", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Need Scheduling, PAs.
+				8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+				log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+				log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+				log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+				log_update_req_s, ktime_get_ns());
+		else if (q->poll_nsec == 0) 	// LHP, EHP
+			write_io_log_verbose("ASNSLE", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Need Scheduling, Lhp, Ehp.
+				8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+				log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+				log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+				log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+				log_update_req_s, ktime_get_ns());
+		else				// CP
+			write_io_log_verbose("ASNSCP", log_cpu_num, log_lba, log_sectors, log_bucket, // After Sleep, Need Scheduling, Classic Polling.
+				8, 8, 8, //log_update_req_r, log_sr_pnlt_r, log_sr_last_r,
+				8, 8, //log_dur_cnt_s, log_dur_cnt_checked_r,
+				8, 8, 8, 8, //log_adj, log_up, log_dn, log_dur,
+				log_sleep_actual_time, log_poll_time, log_poll_ret, log_io_time,
+				log_rq_cpu_num, log_rq_dur, log_rq_dur_cnt,
+				log_dur_cnt_r, log_dur_cnt_checked_s, log_sr_pnlt_s, log_sr_last_s,
+				log_update_req_s, ktime_get_ns());
+	}
 	__set_current_state(TASK_RUNNING);
 	return 0;
 }
@@ -4757,9 +5760,38 @@
 static int __init blk_mq_init(void)
 {
 	int i;
+	struct blk_switch *sc;
+
+	irq_poll_switch = __alloc_percpu(sizeof(struct blk_switch), __alignof__(struct blk_switch));
 
-	for_each_possible_cpu(i)
+	for_each_possible_cpu(i) {
 		init_llist_head(&per_cpu(blk_cpu_done, i));
+                sc = per_cpu_ptr(irq_poll_switch, i);
+                sc->enabled = 0;
+                sc->mode = _PAS;
+                sc->cp_cnt = 0; 
+                sc->pas_cnt = 0; 
+                sc->ol_cnt = 0; 
+                sc->int_cnt = 0; 
+                sc->cp_tot = 0; 
+                sc->pas_tot = 0; 
+                sc->ol_tot = 0; 
+                sc->int_tot = 0;
+		sc->N_POLL = 0;
+		sc->N_INT = 10000;
+		sc->N_PAS = 0; 
+		sc->param1 = 0; // PAS->OL when tf > param1
+		sc->param2 = 10; // OL->PAS when QD <= param2.
+		sc->param3 = 10; // theta (10X) for OL->INT. QD > theta
+		sc->param4 = 1; // 1: enable PAS->CP. 0: disable PAS->CP.
+                sc->ioctr = 0;
+		sc->qd = 0;
+		sc->qd_sum = 0;
+		sc->tf = 0;
+		for(int k=0;k<17;k++)
+			sc->ehpmode[k] = 4;
+		spin_lock_init(&sc->qd_lock);
+	}
 	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
 
 	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
--- linux-5.18/block/blk-sysfs.c	2022-05-23 04:52:31.000000000 +0900
+++ /tmp/kernel-build/linux-5.18/block/blk-sysfs.c	2026-03-09 07:45:13.655326771 +0900
@@ -20,12 +20,685 @@
 #include "blk-cgroup.h"
 #include "blk-throttle.h"
 
+extern void start_io_log(void);
+extern void start_io_log_verbose(void);
+extern void end_io_log(void);
+
 struct queue_sysfs_entry {
 	struct attribute attr;
 	ssize_t (*show)(struct request_queue *, char *);
 	ssize_t (*store)(struct request_queue *, const char *, size_t);
 };
 
+static ssize_t queue_switch_stat_show(struct request_queue *q, char *page)
+{
+	struct blk_switch *sc;
+	int i;
+
+	for_each_possible_cpu(i){
+		sc = per_cpu_ptr(irq_poll_switch, i);
+		printk("CPU[%2d] MODE[%d] QD[%2d] param1: %d param2: %d param3: %d polled io: %llu, pas io: %llu, ol io: %llu, int io: %llu",
+				i, sc->mode, sc->qd, sc->param1, sc->param2, sc->param3, sc->cp_tot, sc->pas_tot, sc->ol_tot, sc->int_tot);
+	}
+
+	return sprintf(page, "\n");
+}
+
+
+static void re_init_pas_stat(struct blk_rq_pas_stat *stat, u32 dur, long long adj, long long up, long long dn)
+{
+	stat->dur = dur;
+	stat->adj = adj;
+	stat->up = up;
+	stat->dn = dn;
+	stat->sr_pnlt = 0;
+	stat->sr_last = 1;
+	stat->update_req = 0;
+	stat->dur_cnt = 1;
+	stat->dur_cnt_checked = 0;
+}
+
+static ssize_t queue_pas_enabled_store(struct request_queue *q, const char *page,
+				size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+
+	if (err < 0)
+		return err;
+
+	if (val == 1 && q->poll_stat){
+		if (q->poll_stat)
+			blk_stat_remove_callback(q, q->poll_cb);
+		blk_stat_free_callback(q->poll_cb);
+	}
+
+	if (val == 0 || val == 1)
+		q->pas_enabled = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t queue_pas_enabled_show(struct request_queue *q, char *page)
+{
+	int val = q->pas_enabled;
+
+	return sprintf(page, "%d\n", val);
+}
+static ssize_t
+queue_switch_param1_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, i;
+	struct blk_switch *sc;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= -1){
+		for_each_possible_cpu(i) {
+                	sc = per_cpu_ptr(irq_poll_switch, i);
+			sc->param1 = val;
+		}
+		q->switch_param1 = val;
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_switch_param1_show(struct request_queue *q, char *page)
+{
+	int val = q->switch_param1;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_switch_param2_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, i;
+	struct blk_switch *sc;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+	if (val >= -1){
+		for_each_possible_cpu(i) {
+                	sc = per_cpu_ptr(irq_poll_switch, i);
+			sc->param2 = val;
+		}
+		q->switch_param2 = val;
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_switch_param2_show(struct request_queue *q, char *page)
+{
+	int val = q->switch_param2;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_switch_param3_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, i;
+	struct blk_switch *sc;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+	if (val >= -1){
+		for_each_possible_cpu(i) {
+                	sc = per_cpu_ptr(irq_poll_switch, i);
+			sc->param3 = val;
+		}
+		q->switch_param3 = val;
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_switch_param3_show(struct request_queue *q, char *page)
+{
+	int val = q->switch_param3;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_switch_param4_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, i;
+	struct blk_switch *sc;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+	if (val >= -1){
+		for_each_possible_cpu(i) {
+                	sc = per_cpu_ptr(irq_poll_switch, i);
+			sc->param4 = val;
+		}
+		q->switch_param4 = val;
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_switch_param4_show(struct request_queue *q, char *page)
+{
+	int val = q->switch_param4;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_switch_enabled_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, i;
+	struct blk_switch *sc;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+	if (val >= -1){
+		for_each_possible_cpu(i) {
+                	sc = per_cpu_ptr(irq_poll_switch, i);
+			sc->enabled = val;
+		}
+		q->switch_enabled = val;
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_switch_enabled_show(struct request_queue *q, char *page)
+{
+	int val = q->switch_enabled;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_pas_adaptive_enabled_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0 || val <= 2)
+		q->pas_adaptive_enabled = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_pas_adaptive_enabled_show(struct request_queue *q, char *page)
+{
+	int val = q->pas_adaptive_enabled;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_ehp_enabled_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+
+	if (err < 0)
+		return err;
+
+	if (val == 1 && q->poll_stat){
+		if (q->poll_stat)
+			blk_stat_remove_callback(q, q->poll_cb);
+		blk_stat_free_callback(q->poll_cb);
+	}
+
+	if (val == 0 || val == 1)
+		q->ehp_enabled = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_ehp_enabled_show(struct request_queue *q, char *page)
+{
+	int val = q->ehp_enabled;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_max_no_lock_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+
+	if (err < 0)
+		return err;
+
+	if (val >= 1)
+		q->max_no_lock = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_max_no_lock_show(struct request_queue *q, char *page)
+{
+	int val = q->max_no_lock;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_poll_threshold_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+
+	if (err < 0)
+		return err;
+
+	if (val >= 0)
+		q->poll_threshold = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_poll_threshold_show(struct request_queue *q, char *page)
+{
+	int val = q->poll_threshold;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_logging_enabled_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val == 0){
+		if (q->logging_enabled){
+			printk("End I/O Log\n");
+			end_io_log();
+			q->logging_enabled = val;
+		}
+	}else if (val == 1){
+		if (q->logging_enabled == 0){
+			printk("Start I/O Log\n");
+			start_io_log();
+			q->logging_enabled = val;
+		}
+	}else if (val == 2){
+		if (q->logging_enabled == 0){
+			printk("Start I/O Log (verbose mode)\n");
+			start_io_log_verbose();
+			q->logging_enabled = val;
+		}
+	}
+	else{
+		return -EINVAL;
+	}
+	return count;
+}
+
+static ssize_t
+queue_logging_enabled_show(struct request_queue *q, char *page)
+{
+	int val = q->logging_enabled;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_d_init_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, bucket_idx, cpu;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 100 && val <= 99000) {
+		q->d_init = val;
+		for_each_possible_cpu(cpu) {
+			struct blk_rq_pas_stat* stat = 
+				per_cpu_ptr(q->pas_stat, cpu);
+			for (bucket_idx=0; bucket_idx<BLK_MQ_POLL_STATS_BKTS; 
+				bucket_idx++)
+				re_init_pas_stat(&stat[bucket_idx], q->d_init, 
+					q->div, q->up_init, q->dn_init);
+		}
+	}
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_d_init_show(struct request_queue *q, char *page)
+{
+	int val = q->d_init;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_up_init_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, bucket_idx, cpu;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0.0001* 1000000 && val <= 0.099 * 1000000) {
+		q->up_init = val;
+		if(q->pas_adaptive_enabled == 1) {
+			q->dn_init = q->up_init * q->updn_ratio;
+			if(q->dn_init > q->max_dn) {
+				q->dn_init = q->max_dn;
+				q->up_init = q->dn_init / q->updn_ratio;
+			}
+			else if(q->dn_init < q->min_dn) {
+				q->dn_init = q->min_dn;
+				q->up_init = q->dn_init / q->updn_ratio;
+			}
+		}
+		for_each_possible_cpu(cpu) {
+			struct blk_rq_pas_stat* stat = 
+				per_cpu_ptr(q->pas_stat, cpu);
+			for (bucket_idx=0; bucket_idx<BLK_MQ_POLL_STATS_BKTS; 
+				bucket_idx++)
+				re_init_pas_stat(&stat[bucket_idx], q->d_init, 
+					q->div + q->up_init, q->up_init, q->dn_init);
+		}
+	} else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_up_init_show(struct request_queue *q, char *page)
+{
+	int val = q->up_init;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_dn_init_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val, bucket_idx, cpu;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0.01 * 1000000 && val <= 0.99 * 1000000) {
+		q->dn_init = val;
+		if(q->pas_adaptive_enabled == 1) {
+			if(q->dn_init > q->max_dn) {
+				q->dn_init = q->max_dn;
+				q->up_init = q->dn_init / q->updn_ratio;
+			}
+			else if(q->dn_init < q->min_dn) {
+				q->dn_init = q->min_dn;
+				q->up_init = q->dn_init / q->updn_ratio;
+			}
+		}
+		
+		for_each_possible_cpu(cpu) {
+			struct blk_rq_pas_stat* stat = 
+				per_cpu_ptr(q->pas_stat, cpu);
+			for (bucket_idx=0; bucket_idx<BLK_MQ_POLL_STATS_BKTS; 
+				bucket_idx++)
+				re_init_pas_stat(&stat[bucket_idx], q->d_init, 
+					q->div + q->up_init, q->up_init, q->dn_init);
+		}
+	} else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_dn_init_show(struct request_queue *q, char *page)
+{
+	int val = q->dn_init;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_heat_up_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0)
+		q->heat_up = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_heat_up_show(struct request_queue *q, char *page)
+{
+	int val = q->heat_up;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_cool_dn_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0 && val < 1 * 1000000)
+		q->cool_dn = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_cool_dn_show(struct request_queue *q, char *page)
+{
+	int val = q->cool_dn;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_min_dn_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0.01 * 1000000 && val <= 0.99 * 1000000)
+		q->min_dn = val;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_min_dn_show(struct request_queue *q, char *page)
+{
+	int val = q->min_dn;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_max_dn_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val >= 0.01 * 1000000 && val <= 0.99 * 1000000)
+		q->max_dn = val;
+	else
+		return -EINVAL;
+
+
+	return count;
+}
+
+static ssize_t
+queue_max_dn_show(struct request_queue *q, char *page)
+{
+	int val = q->max_dn;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t
+queue_pas_exception_store(struct request_queue *q, const char *page, size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val == 1) {
+		q->cnt_rel_hybrid_poll = 0;
+		q->cnt_rel_fops = 0;
+		q->cnt_rel_comp_before_sleep = 0;
+		q->cnt_lock_d_c_separate = 0;
+	} else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t
+queue_pas_exception_show(struct request_queue *q, char *page)
+{
+	int val = 1;
+	printk("hp:%llu,fops:%llu,cbs:%llu,dcs:%llu\n",
+		q->cnt_rel_hybrid_poll, q->cnt_rel_fops,
+		q->cnt_rel_comp_before_sleep, q->cnt_lock_d_c_separate);
+	return sprintf(page, "%d\n", val);
+}
+
 static ssize_t
 queue_var_show(unsigned long var, char *page)
 {
@@ -600,6 +1273,26 @@
 QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
 QUEUE_RW_ENTRY(queue_poll, "io_poll");
 QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
+QUEUE_RW_ENTRY(queue_max_dn, "pas_max_dn");
+QUEUE_RW_ENTRY(queue_min_dn, "pas_min_dn");
+QUEUE_RW_ENTRY(queue_pas_exception, "pas_exception");
+QUEUE_RW_ENTRY(queue_d_init, "pas_d_init");
+QUEUE_RW_ENTRY(queue_up_init, "pas_up_init");
+QUEUE_RW_ENTRY(queue_dn_init, "pas_dn_init");
+QUEUE_RW_ENTRY(queue_cool_dn, "pas_cool_dn");
+QUEUE_RW_ENTRY(queue_heat_up, "pas_heat_up");
+QUEUE_RW_ENTRY(queue_pas_enabled, "pas_enabled");
+QUEUE_RW_ENTRY(queue_pas_adaptive_enabled, "pas_adaptive_enabled");
+QUEUE_RO_ENTRY(queue_switch_stat, "switch_stat");
+QUEUE_RW_ENTRY(queue_ehp_enabled, "ehp_enabled");
+QUEUE_RW_ENTRY(queue_max_no_lock, "pas_max_no_lock");
+QUEUE_RW_ENTRY(queue_poll_threshold, "pas_poll_threshold");
+QUEUE_RW_ENTRY(queue_logging_enabled, "logging_enabled");
+QUEUE_RW_ENTRY(queue_switch_param1, "switch_param1");
+QUEUE_RW_ENTRY(queue_switch_param2, "switch_param2");
+QUEUE_RW_ENTRY(queue_switch_param3, "switch_param3");
+QUEUE_RW_ENTRY(queue_switch_param4, "switch_param4");
+QUEUE_RW_ENTRY(queue_switch_enabled, "switch_enabled");
 QUEUE_RW_ENTRY(queue_wc, "write_cache");
 QUEUE_RO_ENTRY(queue_fua, "fua");
 QUEUE_RO_ENTRY(queue_dax, "dax");
@@ -662,6 +1355,26 @@
 	&queue_dax_entry.attr,
 	&queue_wb_lat_entry.attr,
 	&queue_poll_delay_entry.attr,
+	&queue_pas_enabled_entry.attr,
+	&queue_pas_adaptive_enabled_entry.attr,
+	&queue_switch_stat_entry.attr,
+	&queue_switch_param1_entry.attr,
+	&queue_switch_param2_entry.attr,
+	&queue_switch_param3_entry.attr,
+	&queue_switch_param4_entry.attr,
+	&queue_switch_enabled_entry.attr,
+	&queue_ehp_enabled_entry.attr,
+	&queue_max_no_lock_entry.attr,
+	&queue_poll_threshold_entry.attr,
+	&queue_max_dn_entry.attr,
+	&queue_min_dn_entry.attr,
+	&queue_cool_dn_entry.attr,
+	&queue_heat_up_entry.attr,
+	&queue_d_init_entry.attr,
+	&queue_pas_exception_entry.attr,
+	&queue_up_init_entry.attr,
+	&queue_dn_init_entry.attr,
+	&queue_logging_enabled_entry.attr,
 	&queue_io_timeout_entry.attr,
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 	&blk_throtl_sample_time_entry.attr,
--- linux-5.18/include/linux/blk-mq.h	2022-05-23 04:52:31.000000000 +0900
+++ /tmp/kernel-build/linux-5.18/include/linux/blk-mq.h	2026-03-09 07:45:06.117439159 +0900
@@ -190,6 +190,11 @@
 		u64 fifo_time;
 	};
 
+	int cpu_num;
+	u8 dur_cnt;
+	u64 dur;
+	u64 log_real_sleep_time; /* Used when I/O logging is enabled. */
+
 	/*
 	 * completion callback.
 	 */
@@ -678,6 +683,38 @@
 
 #define BLK_MQ_NO_HCTX_IDX	(-1U)
 
+struct blk_switch {
+	int enabled;
+        int mode; // 0: int, 1: poll, 2: pas, 3: overloaded.
+	int ehpmode[17];
+        int cp_cnt;
+	int pas_cnt;
+	int ol_cnt;
+        int int_cnt;
+	int N_POLL;
+	int N_INT;
+	int N_PAS;
+	int param1; 
+	int param2; 
+	int param3; 
+	int param4; 
+	int ioctr;
+
+	int qd;
+	int qd_sum;
+	int tf;
+
+	spinlock_t qd_lock;
+	atomic_t lock;
+
+	u64 cp_tot;
+	u64 pas_tot;
+	u64 ol_tot;
+	u64 int_tot;
+};
+
+extern struct blk_switch __percpu *irq_poll_switch;
+
 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
 		struct lock_class_key *lkclass);
 #define blk_mq_alloc_disk(set, queuedata)				\
