From 8c07702d9280ef5235bb2bdb1ee472452cffca02 Mon Sep 17 00:00:00 2001
From: zhengchuan <[email protected]>
Date: Wed, 30 Nov 2022 16:47:30 +0800
Subject: [PATCH 6/6] migration/multifd-pin: support migration multifd thread
 pin

support migration multifd thread pin by configuration.

Signed-off-by:zhengchuan<[email protected]>
---
 src/qemu/qemu_migration.c |   2 +
 src/qemu/qemu_process.c   | 119 ++++++++++++++++++++++++++++++++++++++
 src/qemu/qemu_process.h   |   5 ++
 3 files changed, 126 insertions(+)

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index b47e69c642..c2a694eff1 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2534,6 +2534,8 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver,
     priv = vm->privateData;
     priv->origname = g_strdup(origname);
     VIR_FREE(priv->migrationPids);
+    VIR_FREE(priv->migrationMultiFdPids);
+    priv->migrationMultiFdCount = 0;
 
     if (taint_hook) {
         /* Domain XML has been altered by a hook script. */
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index ce1c597da1..0f3e651630 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1629,6 +1629,57 @@ qemuProcessHandleSpiceMigrated(qemuMonitorPtr mon G_GNUC_UNUSED,
 }
 
 
+static void
+qemuProcessHandleMigrationPinStatus(qemuDomainObjPrivatePtr priv, int status)
+{
+    if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT)
+        return;
+
+    switch (status) {
+    case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE:
+    case QEMU_MONITOR_MIGRATION_STATUS_SETUP:
+    case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE:
+    case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER:
+    case QEMU_MONITOR_MIGRATION_STATUS_DEVICE:
+    case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY:
+    case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING:
+    case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED:
+    case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG:
+        break;
+    case QEMU_MONITOR_MIGRATION_STATUS_ERROR:
+        /*
+         * migration thread is still running,
+         * so we can't delete migration Cgroup.
+         */
+        VIR_FREE(priv->migrationPids);
+        VIR_FREE(priv->migrationMultiFdPids);
+        VIR_FREE(priv->migrationThreadPinList);
+        priv->migrationMultiFdCount = 0;
+        virBitmapFree(priv->pcpumap);
+        priv->pcpumap = NULL;
+        break;
+    case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED:
+        VIR_FREE(priv->migrationPids);
+        VIR_FREE(priv->migrationMultiFdPids);
+        VIR_FREE(priv->migrationThreadPinList);
+        priv->migrationMultiFdCount = 0;
+        virBitmapFree(priv->pcpumap);
+        priv->pcpumap = NULL;
+        if (virCgroupDelThread(priv->cgroup,
+                               VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0)
+            VIR_WARN("Failed to delete migration thread Cgroup!");
+        VIR_INFO("success to free pcpumap and migrationPids");
+        break;
+    default:
+        VIR_WARN("got unknown migration status'%s'",
+                qemuMonitorMigrationStatusTypeToString(status));
+        break;
+    }
+
+    return;
+}
+
+
 static int
 qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED,
                                  virDomainObjPtr vm,
@@ -1675,6 +1726,8 @@ qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED,
         }
     }
 
+    qemuProcessHandleMigrationPinStatus(priv, status);
+
  cleanup:
     virObjectUnlock(vm);
     virObjectEventStateQueue(driver->domainEventState, event);
@@ -1915,6 +1968,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
     .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
     .domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded,
     .domainMigrationPid = qemuProcessHandleMigrationPid,
+    .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids,
 };
 
 static void
@@ -2971,6 +3025,71 @@ qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
 }
 
 
+int
+qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+                                      virDomainObjPtr vm,
+                                      int mpid,
+                                      void *opaque G_GNUC_UNUSED)
+{
+    qemuDomainObjPrivatePtr priv;
+    char *mpidOldStr = NULL;
+    char *mpidStr = NULL;
+    virDomainMigrationIDDefPtr migration = NULL;
+    virBitmapPtr pcpumap = NULL;
+    virObjectLock(vm);
+
+    VIR_INFO("Migrating domain %p %s, migration-multifd pid %d",
+              vm, vm->def->name, mpid);
+
+    priv = vm->privateData;
+    if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) {
+        VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job");
+        goto cleanup;
+    }
+
+    if (VIR_ALLOC(migration) < 0) {
+        VIR_ERROR(_("alloc migrationIDDefPtr failure"));
+        goto cleanup;
+    }
+    migration->thread_id = mpid;
+
+    if (qemuProcessSetupMigration(vm, migration) < 0) {
+        VIR_ERROR(_("fail to setup migration multiFd cgroup"));
+        goto cleanup;
+    }
+
+    mpidOldStr = priv->migrationMultiFdPids;
+    if (!mpidOldStr) {
+        mpidStr = g_strdup_printf("%d", mpid);
+    } else {
+        mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid);
+    }
+
+    VIR_FREE(priv->migrationMultiFdPids);
+    priv->migrationMultiFdPids = mpidStr;
+    priv->migrationMultiFdCount++;
+
+    pcpumap = qemuProcessGetPcpumap(priv);
+
+    if (!pcpumap)
+        goto cleanup;
+
+    qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid);
+
+ cleanup:
+    /*
+     * If the value of pcpumap is setted by priv->migrationThreadPinList,
+     * we need to free pcpumap.
+     */
+    if (pcpumap != priv->pcpumap)
+        virBitmapFree(pcpumap);
+    virDomainMigrationIDDefFree(migration);
+    virObjectUnlock(vm);
+
+    return 0;
+}
+
+
 static char *
 qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm)
 {
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index be062b9f95..561795edeb 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -249,3 +249,8 @@ int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
                                   virDomainObjPtr vm,
                                   int mpid,
                                   void *opaque);
+
+int qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+                                          virDomainObjPtr vm,
+                                          int mpid,
+                                          void *opaque);
-- 
2.25.1