From 77a98b652e40ed72dbf8f72be0a17bc7f3b3967b Mon Sep 17 00:00:00 2001
From: liqiang <liqiang64@huawei.com>
Date: Tue, 6 Aug 2024 19:52:06 +0800
Subject: [PATCH 3/3] add compile option of EPOLL_PROXY and fix some problem of
 vsock connection

Signed-off-by: liqiang <liqiang64@huawei.com>
---
 qtfs/qtfs/Makefile             |  3 +++
 qtfs/qtfs/qtfs-mod.c           | 13 ++++++++++++-
 qtfs/qtfs_common/conn.c        | 10 ++++++++++
 qtfs/qtfs_common/misc.c        |  2 ++
 qtfs/qtfs_common/socket.c      |  8 +++++++-
 qtfs/qtfs_common/user_engine.c | 12 +++++++++++-
 qtfs/qtfs_server/Makefile      |  4 ++++
 qtfs/qtfs_server/qtfs-server.c | 12 ++++++++++++
 8 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/qtfs/qtfs/Makefile b/qtfs/qtfs/Makefile
index 9b3ebe5..a6a6dca 100644
--- a/qtfs/qtfs/Makefile
+++ b/qtfs/qtfs/Makefile
@@ -3,6 +3,9 @@ ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ip
 else
 ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ipc/ -I$(src) -DQTFS_CLIENT
 endif
+ifdef EPOLL_PROXY
+ccflags-y += -DEPOLL_PROXY
+endif
 
 KBUILD=/lib/modules/$(shell uname -r)/build/
 COMM=../qtfs_common/
diff --git a/qtfs/qtfs/qtfs-mod.c b/qtfs/qtfs/qtfs-mod.c
index 0ce43fb..79256ad 100644
--- a/qtfs/qtfs/qtfs-mod.c
+++ b/qtfs/qtfs/qtfs-mod.c
@@ -33,8 +33,9 @@ static struct file_system_type qtfs_fs_type = {
 };
 MODULE_ALIAS_FS("qtfs");
 struct kmem_cache *qtfs_inode_priv_cache;
+#ifdef EPOLL_PROXY
 struct task_struct *g_qtfs_epoll_thread = NULL;
-
+#endif
 /*
  * 转发框架层：
  *				1. 调用者先在pvar里预留框架头后，填好自己的私有发送数据。
@@ -124,6 +125,7 @@ retry:
 	return pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV);
 }
 
+#ifdef EPOLL_PROXY
 static int qtfs_epoll_thread(void *data)
 {
 	struct qtfs_conn_var_s *pvar = NULL;
@@ -202,6 +204,7 @@ end:
 	g_qtfs_epoll_thread = NULL;
 	return 0;
 }
+#endif
 
 struct file_operations qtfs_misc_fops = {
 	.owner=THIS_MODULE,
@@ -236,12 +239,14 @@ static int __init qtfs_init(void)
 		goto conn_init_err;
 	}
 	qtfs_whitelist_initset();
+#ifdef EPOLL_PROXY
 	g_qtfs_epoll_thread = kthread_run(qtfs_epoll_thread, NULL, "qtfs_epoll");
 	if (IS_ERR_OR_NULL(g_qtfs_epoll_thread)) {
 		qtfs_err("qtfs epoll thread run failed.\n");
 		ret = QTFS_PTR_ERR(g_qtfs_epoll_thread);
 		goto epoll_thread_err;
 	}
+#endif
 	qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL);
 	if (qtfs_diag_info == NULL) {
 		qtfs_err("kmalloc qtfs diag info failed.");
@@ -278,7 +283,9 @@ syscall_replace_err:
 misc_register_err:
 	kfree(qtfs_diag_info);
 diag_malloc_err:
+#ifdef EPOLL_PROXY
 	kthread_stop(g_qtfs_epoll_thread);
+#endif
 epoll_thread_err:
 	qtfs_conn_param_fini();
 conn_init_err:
@@ -293,18 +300,22 @@ static void __exit qtfs_exit(void)
 	int ret;
 	qtfs_mod_exiting = true;
 
+#ifdef EPOLL_PROXY
 	if (g_qtfs_epoll_thread) {
 		kthread_stop(g_qtfs_epoll_thread);
 	}
+#endif
 
 	qtfs_conn_param_fini();
 	qtfs_misc_destroy();
+#ifdef EPOLL_PROXY
 	if (qtfs_epoll_var != NULL) {
 		qtfs_epoll_cut_conn(qtfs_epoll_var);
 		qtfs_epoll_var->conn_ops->conn_var_fini(qtfs_epoll_var);
 		kfree(qtfs_epoll_var);
 		qtfs_epoll_var = NULL;
 	}
+#endif
 	qtfs_whitelist_exit();
 
 	kfree(qtfs_diag_info);
diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c
index 5f080cd..fd5f654 100644
--- a/qtfs/qtfs_common/conn.c
+++ b/qtfs/qtfs_common/conn.c
@@ -42,7 +42,9 @@ static struct mutex g_param_mutex;
 static struct mutex g_fifo_mutex;
 int qtfs_mod_exiting = false;
 struct qtfs_conn_var_s *qtfs_thread_var[QTFS_MAX_THREADS] = {NULL};
+#ifdef EPOLL_PROXY
 struct qtfs_conn_var_s *qtfs_epoll_var = NULL;
+#endif
 #ifdef QTFS_SERVER
 struct qtfs_server_userp_s *qtfs_userps = NULL;
 #endif
@@ -520,7 +522,11 @@ void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir)
 static int qtfs_sm_connecting(struct qtfs_conn_var_s *pvar)
 {
 	int ret = QTERROR;
+#ifdef QTFS_CLIENT
 	int retry = 3;
+#else
+	int retry = 1;
+#endif
 	while (qtfs_mod_exiting == false && retry-- > 0) {
 		ret = pvar->conn_ops->conn_new_connection(&pvar->conn_var, pvar->user_type);
 		if (ret == 0) {
@@ -873,6 +879,7 @@ retry:
 	return pvar;
 }
 
+#ifdef EPOLL_PROXY
 struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void)
 {
 	struct qtfs_conn_var_s *pvar = NULL;
@@ -920,6 +927,7 @@ struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void)
 	qtfs_info("qtfs create new epoll param state:%s", QTCONN_CUR_STATE(pvar));
 	return pvar;
 }
+#endif
 
 void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar)
 {
@@ -944,6 +952,7 @@ void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar)
 	mutex_unlock(&g_param_mutex);
 }
 
+#ifdef EPOLL_PROXY
 void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar)
 {
 	int ret = 0;
@@ -958,6 +967,7 @@ void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar)
 		qtfs_err("qtfs epoll put param exit failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar));
 	}
 }
+#endif
 
 #ifdef QTFS_CLIENT
 /* fifo的机制有所不同，每一个pvar对应唯一一个fifo的访问，生命周期贯穿
diff --git a/qtfs/qtfs_common/misc.c b/qtfs/qtfs_common/misc.c
index e1c3520..3adfb57 100644
--- a/qtfs/qtfs_common/misc.c
+++ b/qtfs/qtfs_common/misc.c
@@ -68,7 +68,9 @@ void qtfs_misc_flush_threadstate(void)
 		}
 		qtfs_diag_info->thread_state[i] = qtfs_thread_var[i]->state;
 	}
+#ifdef EPOLL_PROXY
 	qtfs_diag_info->epoll_state = (qtfs_epoll_var == NULL) ? -1 : qtfs_epoll_var->state;
+#endif
 }
 
 void qtfs_req_size(void)
diff --git a/qtfs/qtfs_common/socket.c b/qtfs/qtfs_common/socket.c
index f247be0..233a1ce 100644
--- a/qtfs/qtfs_common/socket.c
+++ b/qtfs/qtfs_common/socket.c
@@ -162,6 +162,11 @@ static int qtfs_conn_sock_server_accept(void *connvar, qtfs_conn_type_e type)
 		mutex_unlock(&qtfs_server_main_sock[type].lock);
 		return -EFAULT;
 	}
+	if (qtfs_server_thread_run == 0) {
+		qtfs_err("engine is exit now, return right now.");
+		mutex_unlock(&qtfs_server_main_sock[type].lock);
+		return -EINTR;
+	}
 	sock = qtfs_server_main_sock[type].sock;
 
 	if (sock == NULL) {
@@ -246,7 +251,7 @@ static int qtfs_conn_sock_init(void *connvar, qtfs_conn_type_e type)
 	}
 
 	qtfs_info("qtfs socket init sock OK, type:%d!", type);
-	qtfs_sock_recvtimeo_set(sock, 0, 100000);
+	qtfs_sock_recvtimeo_set(sock, 1, 0);
 	qtfs_server_main_sock[type].sock = sock;
 	mutex_unlock(&qtfs_server_main_sock[type].lock);
 	return 0;
@@ -282,6 +287,7 @@ static int qtfs_conn_sock_client_connect(void *connvar, qtfs_conn_type_e type)
 
 	ret = sock->ops->connect(sock, (struct sockaddr *)&saddr, sizeof(saddr), 0);
 	if (ret < 0) {
+		qtfs_err("qtfs client connect failed, ret:%d", ret);
 		return ret;
 	}
 #ifdef QTFS_TEST_MODE
diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c
index a3edac2..a602980 100644
--- a/qtfs/qtfs_common/user_engine.c
+++ b/qtfs/qtfs_common/user_engine.c
@@ -177,8 +177,8 @@ static void *qtfs_engine_kthread(void *arg)
 			break;
 		}
 		if (sig_int_flag == 1) {
+			sig_int_flag = 0;
 			engine_out("qtfs engine recv SIGINT.");
-
 			if (qtfs_fd < 0) {
 				engine_err("qtfs engine signal int file:%s open failed, fd:%d.", QTFS_SERVER_FILE, qtfs_fd);
 				continue;
@@ -204,6 +204,7 @@ static void qtfs_signal_int(int signum)
 	return;
 }
 
+#ifdef EPOLL_PROXY
 static void *qtfs_engine_epoll_thread(void *data)
 {
 	struct engine_arg *arg = (struct engine_arg *)data;
@@ -264,6 +265,7 @@ int qtfs_epoll_init(int fd)
 
 	return epfd;
 }
+#endif
 
 static void qtfs_whitelist_free_items(char **items, gsize count)
 {
@@ -498,11 +500,13 @@ int main(int argc, char *argv[])
 	}
 	qtfs_fd = fd;
 	// init epoll
+#ifdef EPOLL_PROXY
 	int epfd = qtfs_epoll_init(fd);
 	if (epfd < 0) {
 		close(fd);
 		return -1;
 	}
+#endif
 	ret = ioctl(fd, QTFS_IOCTL_EXIT, 1);
 	if (ret == QTERROR) {
 		goto end;
@@ -539,7 +543,9 @@ int main(int argc, char *argv[])
 		arg[i].thread_idx = i;
 		(void)pthread_create(&texec[i], NULL, qtfs_engine_kthread, &arg[i]);
 	}
+#ifdef EPOLL_PROXY
 	(void)pthread_create(&tepoll, NULL, qtfs_engine_epoll_thread, &arg[0]);
+#endif
 
 #ifdef QTFS_TEST_MODE
 	fifo_arg.addr = argv[3];
@@ -566,14 +572,18 @@ int main(int argc, char *argv[])
 		pthread_join(texec[i], NULL);
 		engine_out("qtfs engine join thread %d.", i);
 	}
+#ifdef EPOLL_PROXY
 	pthread_join(tepoll, NULL);
+#endif
 	pthread_join(tfifo, NULL);
 engine_free:
 	qtfs_engine_userp_free(userp, thread_nums);
 	engine_out("qtfs engine join epoll thread.");
 end:
 	(void)ioctl(fd, QTFS_IOCTL_EXIT, 0);
+#ifdef EPOLL_PROXY
 	close(epfd);
+#endif
 	close(fd);
 	engine_out("qtfs engine over.");
 	return ret;
diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile
index 4608365..765567a 100644
--- a/qtfs/qtfs_server/Makefile
+++ b/qtfs/qtfs_server/Makefile
@@ -4,6 +4,10 @@ CFLAGS += -DUDS_TEST_MODE -DQTFS_TEST_MODE
 else
 ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER
 endif
+ifdef EPOLL_PROXY
+ccflags-y += -DEPOLL_PROXY
+CFLAGS += -DEPOLL_PROXY
+endif
 
 CFLAGS += -g -O2
 CFLAGS += -fstack-protector-strong
diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c
index f4e201c..48248ba 100644
--- a/qtfs/qtfs_server/qtfs-server.c
+++ b/qtfs/qtfs_server/qtfs-server.c
@@ -59,6 +59,7 @@ void qtfs_server_epoll_exit(void)
 	return;
 }
 
+#ifdef EPOLL_PROXY
 long qtfs_server_epoll_thread(struct qtfs_conn_var_s *pvar)
 {
 	int n;
@@ -170,6 +171,7 @@ long qtfs_server_epoll_init(void)
 
 	return QTOK;
 }
+#endif
 
 int qtfs_server_fd_bitmap_init(void)
 {
@@ -308,11 +310,16 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			write_unlock(&qtfs_epoll_rwlock);
 			break;
 		case QTFS_IOCTL_EPOLL_THREAD_INIT:
+#ifdef EPOLL_PROXY
 			write_lock(&qtfs_epoll_rwlock);
 			ret = qtfs_server_epoll_init();
 			write_unlock(&qtfs_epoll_rwlock);
+#else
+			qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it.");
+#endif
 			break;
 		case QTFS_IOCTL_EPOLL_THREAD_RUN:
+#ifdef EPOLL_PROXY
 			write_lock(&qtfs_epoll_rwlock);
 			if (qtfs_epoll_var == NULL) {
 				qtfs_err("qtfs epoll thread run failed, var is invalid.");
@@ -326,6 +333,9 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a
 				qtfs_info("qtfs epoll thread exit.");
 				qtfs_epoll_cut_conn(qtfs_epoll_var);
 			}
+#else
+			qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it.");
+#endif
 			break;
 		case QTFS_IOCTL_EXIT:
 			if (arg != 0 && arg != 1) {
@@ -403,6 +413,7 @@ static void __exit qtfs_server_exit(void)
 
 	qtfs_conn_param_fini();
 
+#ifdef EPOLL_PROXY
 	if (qtfs_epoll_var != NULL) {
 		qtfs_epoll_cut_conn(qtfs_epoll_var);
 		qtfs_conn_fini(qtfs_epoll_var);
@@ -410,6 +421,7 @@ static void __exit qtfs_server_exit(void)
 		kfree(qtfs_epoll_var);
 		qtfs_epoll_var = NULL;
 	}
+#endif
 	if (qtfs_diag_info != NULL) {
 		kfree(qtfs_diag_info);
 		qtfs_diag_info = NULL;
-- 
2.37.1 (Apple Git-137.1)