From c2cfb702946d01480236b34c1dacaeca0758cff0 Mon Sep 17 00:00:00 2001
From: liqiang <[email protected]>
Date: Fri, 22 Nov 2024 09:31:03 +0800
Subject: [PATCH] fix rwlock cause schedule bug if pagefault or wait for read
 file page

Signed-off-by: liqiang <[email protected]>
---
 qtfs/include/conn.h            |  2 +-
 qtfs/qtfs_common/conn.c        |  4 +--
 qtfs/qtfs_common/misc.c        | 24 +++++++-------
 qtfs/qtfs_server/fsops.c       |  8 ++---
 qtfs/qtfs_server/qtfs-server.c | 57 +++++++++++++++++-----------------
 qtfs/qtfs_server/qtfs-server.h |  2 +-
 6 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/qtfs/include/conn.h b/qtfs/include/conn.h
index 5f74a46..aaf746c 100644
--- a/qtfs/include/conn.h
+++ b/qtfs/include/conn.h
@@ -206,7 +206,7 @@ struct qtfs_wl_cap {
 };
 
 struct qtfs_wl {
-	rwlock_t rwlock;
+	struct rw_semaphore rwlock;
 	struct qtfs_wl_cap cap[QTFS_WHITELIST_MAX];
 };
 
diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c
index fd5f654..1569bf8 100644
--- a/qtfs/qtfs_common/conn.c
+++ b/qtfs/qtfs_common/conn.c
@@ -116,7 +116,7 @@ static int qtfs_uds_remote_whitelist(char *path)
 	int i;
 	int ret = 1;
 	struct qtfs_wl_cap *cap;
-	read_lock(&g_qtfs_wl.rwlock);
+	down_read(&g_qtfs_wl.rwlock);
 	cap = &g_qtfs_wl.cap[QTFS_WHITELIST_UDSCONNECT];
 	for (i = 0; i < cap->nums; i++) {
 		if (qtfs_white_list_match(path, cap->item[i], strlen(cap->item[i]), WHITELIST_MATCH_PREFIX)) {
@@ -124,7 +124,7 @@ static int qtfs_uds_remote_whitelist(char *path)
 			break;
 		}
 	}
-	read_unlock(&g_qtfs_wl.rwlock);
+	up_read(&g_qtfs_wl.rwlock);
 	return ret;
 }
 
diff --git a/qtfs/qtfs_common/misc.c b/qtfs/qtfs_common/misc.c
index 3adfb57..1c57a85 100644
--- a/qtfs/qtfs_common/misc.c
+++ b/qtfs/qtfs_common/misc.c
@@ -137,7 +137,7 @@ void qtfs_req_size(void)
 void qtfs_whitelist_initset(void)
 {
 	int type;
-	rwlock_init(&g_qtfs_wl.rwlock);
+	init_rwsem(&g_qtfs_wl.rwlock);
 	for (type = 0; type < QTFS_WHITELIST_MAX; type++) {
 		memset(&g_qtfs_wl.cap[type], 0, sizeof(struct qtfs_wl_cap));
 	}
@@ -158,7 +158,7 @@ static int qtfs_whitelist_add(struct qtfs_wl_item *uitem)
 {
 	// uitem->type is checked
 	struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type];
-	write_lock(&g_qtfs_wl.rwlock);
+	down_write(&g_qtfs_wl.rwlock);
 	if (cap->nums >= QTFS_WL_MAX_NUM) {
 		qtfs_err("qtfs add white list failed, nums:%u reach upper limit:%d", cap->nums, QTFS_WL_MAX_NUM);
 		goto err_end;
@@ -178,11 +178,11 @@ static int qtfs_whitelist_add(struct qtfs_wl_item *uitem)
 	}
 	qtfs_info("Successed to add white list type:%u len:%u path:[%s]", uitem->type, uitem->len, cap->item[cap->nums]);
 	cap->nums++;
-	write_unlock(&g_qtfs_wl.rwlock);
+	up_write(&g_qtfs_wl.rwlock);
 	return 0;
 
 err_end:
-	write_unlock(&g_qtfs_wl.rwlock);
+	up_write(&g_qtfs_wl.rwlock);
 	return -1;
 }
 
@@ -190,7 +190,7 @@ static int qtfs_whitelist_del(struct qtfs_wl_item *uitem)
 {
 	// type is checked
 	struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type];
-	write_lock(&g_qtfs_wl.rwlock);
+	down_write(&g_qtfs_wl.rwlock);
 	if (uitem->index >= cap->nums) {
 		qtfs_err("White list del type:%u nums:%u, invalid index:%u", uitem->type, cap->nums, uitem->index);
 		goto err_end;
@@ -206,10 +206,10 @@ static int qtfs_whitelist_del(struct qtfs_wl_item *uitem)
 	qtfs_info("white list del type:%u total nums:%u delindex:%u", uitem->type, cap->nums, uitem->index);
 
 	cap->nums--;
-	write_unlock(&g_qtfs_wl.rwlock);
+	up_write(&g_qtfs_wl.rwlock);
 	return 0;
 err_end:
-	write_unlock(&g_qtfs_wl.rwlock);
+	up_write(&g_qtfs_wl.rwlock);
 	return -1;
 }
 
@@ -218,7 +218,7 @@ static int qtfs_whitelist_get(struct qtfs_wl_item *uitem)
 	// type is checked
 	struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type];
 	int len;
-	read_lock(&g_qtfs_wl.rwlock);
+	down_read(&g_qtfs_wl.rwlock);
 	if (uitem->index >= cap->nums) {
 		qtfs_err("query white list invalid index:%u type:%u total nums:%u", uitem->index, uitem->type, cap->nums);
 		goto err_end;
@@ -233,11 +233,11 @@ static int qtfs_whitelist_get(struct qtfs_wl_item *uitem)
 		goto err_end;
 	}
 	qtfs_info("white list query type:%u total nums:%u index:%u len:%d", uitem->type, cap->nums, uitem->index, len);
-	read_unlock(&g_qtfs_wl.rwlock);
+	up_read(&g_qtfs_wl.rwlock);
 	return 0;
 
 err_end:
-	read_unlock(&g_qtfs_wl.rwlock);
+	up_read(&g_qtfs_wl.rwlock);
 	return -1;
 }
 
@@ -246,7 +246,7 @@ void qtfs_whitelist_clearall(void)
 	struct qtfs_wl_cap *cap = NULL;
 	int type;
 	int item;
-	write_lock(&g_qtfs_wl.rwlock);
+	down_write(&g_qtfs_wl.rwlock);
 	for (type = 0; type < QTFS_WHITELIST_MAX; type++) {
 		cap = &g_qtfs_wl.cap[type];
 		for (item = 0; item < cap->nums; item++) {
@@ -255,7 +255,7 @@ void qtfs_whitelist_clearall(void)
 		}
 		cap->nums = 0;
 	}
-	write_unlock(&g_qtfs_wl.rwlock);
+	up_write(&g_qtfs_wl.rwlock);
 	return;
 }
 
diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c
index f2f3749..f0dce2c 100644
--- a/qtfs/qtfs_server/fsops.c
+++ b/qtfs/qtfs_server/fsops.c
@@ -62,7 +62,7 @@ static bool _in_white_list(char *path, int type, int match_type)
 		return false;
 	}
 
-	read_lock(&g_qtfs_wl.rwlock);
+	down_read(&g_qtfs_wl.rwlock);
 	cap = &g_qtfs_wl.cap[type];
 	for (i = 0; i < cap->nums; i++) {
 		if (qtfs_white_list_match(path, cap->item[i], strlen(cap->item[i]), match_type)) {
@@ -70,7 +70,7 @@ static bool _in_white_list(char *path, int type, int match_type)
 			break;
 		}
 	}
-	read_unlock(&g_qtfs_wl.rwlock);
+	up_read(&g_qtfs_wl.rwlock);
 	return in_wl != -1;
 }
 
@@ -1736,12 +1736,12 @@ int qtfs_conn_server_run(struct qtfs_conn_var_s *pvar)
 				qtfs_err("qtfs server req type:%u precheck failed.", req->type);
 				goto out;
 			}
-			read_lock(&g_userp_rwlock);
+			down_read(&g_userp_rwlock);
 			arg.userp = &qtfs_userps[pvar->cur_threadidx];
 			if (arg.userp->userp == NULL || arg.userp->userp2 == NULL)
 				qtfs_err("server run userp or userp2 is invalid");
 			rsp->len = qtfs_server_handles[req->type].handle(&arg);
-			read_unlock(&g_userp_rwlock);
+			up_read(&g_userp_rwlock);
 			rsp->type = req->type;
 			rsp->err = QTFS_OK;
 			totalproc++;
diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c
index 48248ba..f7907ca 100644
--- a/qtfs/qtfs_server/qtfs-server.c
+++ b/qtfs/qtfs_server/qtfs-server.c
@@ -28,7 +28,7 @@
 #define QTFS_EPOLL_RETRY_INTVL 500 // unit ms
 
 int qtfs_server_thread_run = 1;
-DEFINE_RWLOCK(g_userp_rwlock);
+struct rw_semaphore g_userp_rwlock;
 struct qtserver_fd_bitmap qtfs_fd_bitmap;
 
 long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
@@ -44,18 +44,18 @@ struct qtfs_server_epoll_s qtfs_epoll = {
 	.events = NULL,
 	.kevents = NULL,
 };
-rwlock_t qtfs_epoll_rwlock;
+struct rw_semaphore qtfs_epoll_rwlock;
 
 void qtfs_server_epoll_exit(void)
 {
-	write_lock(&qtfs_epoll_rwlock);
+	down_write(&qtfs_epoll_rwlock);
 	if (qtfs_epoll.kevents == NULL) {
-		write_unlock(&qtfs_epoll_rwlock);
+		up_write(&qtfs_epoll_rwlock);
 		return;
 	}
 	kfree(qtfs_epoll.kevents);
 	qtfs_epoll.kevents = NULL;
-	write_unlock(&qtfs_epoll_rwlock);
+	up_write(&qtfs_epoll_rwlock);
 	return;
 }
 
@@ -211,35 +211,35 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			} else {
 				qtfs_conn_put_param(pvar);
 			}
-			if (!write_trylock(&g_userp_rwlock)) {
+			if (!down_write_trylock(&g_userp_rwlock)) {
 				qtfs_err("try lock userps failed.");
 				return QTERROR;
 			}
 			if (qtfs_server_fd_bitmap_init() < 0) {
 				qtfs_err("fd bitmap init failed.");
-				write_unlock(&g_userp_rwlock);
+				up_write(&g_userp_rwlock);
 				return QTERROR;
 			}
 			if (copy_from_user(&init_userp, (void __user *)arg, sizeof(struct qtfs_thread_init_s))) {
 				qtfs_err("qtfs ioctl thread init copy from user failed.");
-				write_unlock(&g_userp_rwlock);
+				up_write(&g_userp_rwlock);
 				return QTERROR;
 			}
 			if (qtfs_userps == NULL || init_userp.thread_nums > QTFS_MAX_THREADS || init_userp.thread_nums == 0) {
 				qtfs_err("qtfs ioctl thread init userps invalid thread nums:%d.", init_userp.thread_nums);
-				write_unlock(&g_userp_rwlock);
+				up_write(&g_userp_rwlock);
 				return QTERROR;
 			}
 			memset(qtfs_userps, 0, QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s));
 			if (init_userp.thread_nums > QTFS_MAX_THREADS) {
 				qtfs_err("qtfs ioctl thread init invalid input thread_num:%d", init_userp.thread_nums);
-				write_unlock(&g_userp_rwlock);
+				up_write(&g_userp_rwlock);
 				return QTERROR;
 			}
 			if (copy_from_user(qtfs_userps, (void __user *)init_userp.userp,
 							init_userp.thread_nums * sizeof(struct qtfs_server_userp_s))) {
 				qtfs_err("qtfs ioctl thread init copy from userp failed.");
-				write_unlock(&g_userp_rwlock);
+				up_write(&g_userp_rwlock);
 				return QTERROR;
 			}
 			for (i = 0; i < init_userp.thread_nums; i++) {
@@ -248,12 +248,12 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 					!access_ok(qtfs_userps[i].userp2, qtfs_userps[i].size)) {
 					qtfs_err("userp set failed");
 					ret = QTERROR;
-					write_unlock(&g_userp_rwlock);
+					up_write(&g_userp_rwlock);
 					break;
 				}
 				qtfs_info("userp set idx:%d size:%lu", i, qtfs_userps[i].size);
 			}
-			write_unlock(&g_userp_rwlock);
+			up_write(&g_userp_rwlock);
 			break;
 		case QTFS_IOCTL_THREAD_RUN:
 			pvar = qtfs_conn_get_param();
@@ -268,7 +268,7 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			qtfs_conn_put_param(pvar);
 			break;
 		case QTFS_IOCTL_EPFDSET:
-			write_lock(&qtfs_epoll_rwlock);
+			down_write(&qtfs_epoll_rwlock);
 			if (qtfs_epoll.kevents != NULL) {
 				kfree(qtfs_epoll.kevents);
 				qtfs_epoll.kevents = NULL;
@@ -276,25 +276,25 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			if (copy_from_user(&qtfs_epoll, (void __user *)arg, sizeof(struct qtfs_server_epoll_s))) {
 				qtfs_err("copy epoll struct from arg failed.");
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
 			if (qtfs_epoll.event_nums > QTFS_MAX_EPEVENTS_NUM || qtfs_epoll.event_nums == 0) {
 				qtfs_err("epoll arg set failed, event nums:%d too big", qtfs_epoll.event_nums);
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
 			if (qtfs_epoll.epfd < 3) {
 				qtfs_err("epoll epfd set failed, epfd:%d should be greater than 2", qtfs_epoll.epfd);
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
 			if (!access_ok(qtfs_epoll.events, qtfs_epoll.event_nums * sizeof(struct epoll_event))) {
 				qtfs_err("epoll events set failed, check pointer of qtfs_epoll.events failed");
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
 			qtfs_info("epoll arg set, epfd:%d event nums:%d events.",
@@ -304,31 +304,31 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			if (qtfs_epoll.kevents == NULL) {
 				qtfs_err("epoll kernel events kmalloc failed.");
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
-			write_unlock(&qtfs_epoll_rwlock);
+			up_write(&qtfs_epoll_rwlock);
 			break;
 		case QTFS_IOCTL_EPOLL_THREAD_INIT:
 #ifdef EPOLL_PROXY
-			write_lock(&qtfs_epoll_rwlock);
+			down_write(&qtfs_epoll_rwlock);
 			ret = qtfs_server_epoll_init();
-			write_unlock(&qtfs_epoll_rwlock);
+			up_write(&qtfs_epoll_rwlock);
 #else
 			qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it.");
 #endif
 			break;
 		case QTFS_IOCTL_EPOLL_THREAD_RUN:
 #ifdef EPOLL_PROXY
-			write_lock(&qtfs_epoll_rwlock);
+			down_write(&qtfs_epoll_rwlock);
 			if (qtfs_epoll_var == NULL) {
 				qtfs_err("qtfs epoll thread run failed, var is invalid.");
 				ret = QTERROR;
-				write_unlock(&qtfs_epoll_rwlock);
+				up_write(&qtfs_epoll_rwlock);
 				break;
 			}
 			ret = qtfs_server_epoll_thread(qtfs_epoll_var);
-			write_unlock(&qtfs_epoll_rwlock);
+			up_write(&qtfs_epoll_rwlock);
 			if (ret == QTEXIT) {
 				qtfs_info("qtfs epoll thread exit.");
 				qtfs_epoll_cut_conn(qtfs_epoll_var);
@@ -367,9 +367,10 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 static int __init qtfs_server_init(void)
 {
 	qtfs_log_init(qtfs_log_level, sizeof(qtfs_log_level));
+	init_rwsem(&g_userp_rwlock);
 	if (qtfs_kallsyms_hack_init() != 0)
 		return -1;
-	rwlock_init(&qtfs_epoll_rwlock);
+	init_rwsem(&qtfs_epoll_rwlock);
 	qtfs_whitelist_initset();
 
 	qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL);
@@ -426,7 +427,7 @@ static void __exit qtfs_server_exit(void)
 		kfree(qtfs_diag_info);
 		qtfs_diag_info = NULL;
 	}
-	write_lock(&g_userp_rwlock);
+	down_write(&g_userp_rwlock);
 	if (qtfs_userps != NULL) {
 		kfree(qtfs_userps);
 		qtfs_userps = NULL;
@@ -436,7 +437,7 @@ static void __exit qtfs_server_exit(void)
 		qtfs_fd_bitmap.bitmap = NULL;
 		qtfs_fd_bitmap.nbits = 0;
 	}
-	write_unlock(&g_userp_rwlock);
+	up_write(&g_userp_rwlock);
 	qtfs_whitelist_exit();
 	qtfs_server_epoll_exit();
 
diff --git a/qtfs/qtfs_server/qtfs-server.h b/qtfs/qtfs_server/qtfs-server.h
index 8135dbb..fa49ad9 100644
--- a/qtfs/qtfs_server/qtfs-server.h
+++ b/qtfs/qtfs_server/qtfs-server.h
@@ -17,7 +17,7 @@
 extern int qtfs_server_thread_run;
 extern struct qtfs_server_epoll_s qtfs_epoll;
 extern int qtfs_mod_exiting;
-extern rwlock_t g_userp_rwlock;
+extern struct rw_semaphore g_userp_rwlock;
 extern struct qtserver_fd_bitmap qtfs_fd_bitmap;
 
 struct qtserver_arg {
-- 
2.39.2 (Apple Git-143)