From 23ab7175e64ab4d75fbcb6874008843cc78b65b8 Mon Sep 17 00:00:00 2001
From: Ashish Pandey <[email protected]>
Date: Fri, 16 Apr 2021 18:48:56 +0530
Subject: [PATCH 541/542] glusterd-volgen: Add functionality to accept any
 custom xlator

Add new function which allow users to insert any custom xlators.
It makes to provide a way to add any processing into file operations.

Users can deploy the plugin(xlator shared object) and integrate it to glusterfsd.

If users want to enable a custom xlator, do the follows:

1. put xlator object(.so file) into "XLATOR_DIR/user/"
2. set the option user.xlator.<xlator> to the existing xlator-name to specify of the position in graph
3. restart gluster volume

Options for custom xlator are able to set in "user.xlator.<xlator>.<optkey>".

Backport of :
>https://github.com/gluster/glusterfs/commit/ea86b664f3b1f54901ce1b7d7fba7d80456f2089
>Fixes: https://github.com/gluster/glusterfs/issues/1943
>Change-Id: Ife3ae1514ea474f5dae2897223012f9d04b64674
>Signed-off-by:Ryo Furuhashi <[email protected]>
>Co-authored-by: Yaniv Kaul <[email protected]>
>Co-authored-by: Xavi Hernandez <[email protected]>

Change-Id: Ic8f28bfcfde67213eb1092b0ebf4822c874d37bb
BUG: 1927235
Signed-off-by: Ashish Pandey <[email protected]>
Reviewed-on: https://code.engineering.redhat.com/gerrit/236830
Tested-by: RHGS Build Bot <[email protected]>
Reviewed-by: Ravishankar Narayanankutty <[email protected]>
Reviewed-by: Xavi Hernandez Juan <[email protected]>
---
 cli/src/cli-rpc-ops.c                       | 148 ++++++++++++++++++++------
 cli/src/cli.h                               |   2 -
 tests/basic/user-xlator.t                   |  65 ++++++++++++
 tests/env.rc.in                             |   3 +
 xlators/mgmt/glusterd/src/glusterd-volgen.c | 155 ++++++++++++++++++++++++++++
 5 files changed, 342 insertions(+), 31 deletions(-)
 create mode 100755 tests/basic/user-xlator.t

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 4e91265..51b5447 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -2269,49 +2269,131 @@ out:
     return ret;
 }
 
-char *
-is_server_debug_xlator(void *myframe)
+/*
+ * returns
+ *   1 : is server debug xlator
+ *   0 : is not server debug xlator
+ *  <0 : error
+ */
+static int
+is_server_debug_xlator(char *key, char *value)
+{
+    if (!key || !value)
+        return -1;
+
+    if (strcmp("debug.trace", key) == 0 ||
+        strcmp("debug.error-gen", key) == 0) {
+        if (strcmp("client", value) == 0)
+            return 0;
+        else
+            return 1;
+    }
+
+    return 0;
+}
+
+/*
+ * returns
+ *   1 : is user xlator
+ *   0 : is not user xlator
+ *  <0 : error
+ */
+static int
+is_server_user_xlator(char *key, char *value)
+{
+    int ret = 0;
+
+    if (!key || !value)
+        return -1;
+
+    ret = fnmatch("user.xlator.*", key, 0);
+    if (ret < 0) {
+        ret = -1;
+        goto out;
+    } else if (ret == FNM_NOMATCH) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = fnmatch("user.xlator.*.*", key, 0);
+    if (ret < 0) {
+        ret = -1;
+        goto out;
+    } else if (ret != FNM_NOMATCH) {  // this is user xlator's option key
+        ret = 0;
+        goto out;
+    }
+
+    ret = 1;
+
+out:
+    return ret;
+}
+
+static int
+added_server_xlator(void *myframe, char **added_xlator)
 {
     call_frame_t *frame = NULL;
     cli_local_t *local = NULL;
     char **words = NULL;
     char *key = NULL;
     char *value = NULL;
-    char *debug_xlator = NULL;
+    int ret = 0;
 
     frame = myframe;
     local = frame->local;
     words = (char **)local->words;
 
     while (*words != NULL) {
-        if (strstr(*words, "trace") == NULL &&
-            strstr(*words, "error-gen") == NULL) {
-            words++;
-            continue;
-        }
-
         key = *words;
         words++;
         value = *words;
-        if (value == NULL)
+
+        if (!value) {
             break;
-        if (strstr(value, "client")) {
-            words++;
-            continue;
-        } else {
-            if (!(strstr(value, "posix") || strstr(value, "acl") ||
-                  strstr(value, "locks") || strstr(value, "io-threads") ||
-                  strstr(value, "marker") || strstr(value, "index"))) {
-                words++;
-                continue;
-            } else {
-                debug_xlator = gf_strdup(key);
-                break;
+        }
+
+        ret = is_server_debug_xlator(key, value);
+        if (ret < 0) {
+            gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+                   "failed to check that debug xlator was added");
+            ret = -1;
+            goto out;
+        }
+
+        if (ret) {
+            *added_xlator = gf_strdup(key);
+            if (!*added_xlator) {
+                gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+                       "Out of memory");
+                ret = -1;
+                goto out;
+            }
+            break;
+        }
+
+        ret = is_server_user_xlator(key, value);
+        if (ret < 0) {
+            gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+                   "failed to check that user xlator was added");
+            ret = -1;
+            goto out;
+        }
+
+        if (ret) {
+            *added_xlator = gf_strdup(key);
+            if (!*added_xlator) {
+                gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+                       "Out of memory");
+                ret = -1;
+                goto out;
             }
+            break;
         }
     }
 
-    return debug_xlator;
+out:
+    return ret;
 }
 
 int
@@ -2327,7 +2409,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
     char msg[1024] = {
         0,
     };
-    char *debug_xlator = NULL;
+    char *added_xlator = NULL;
     char tmp_str[512] = {
         0,
     };
@@ -2365,18 +2447,26 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
      * The process has to be restarted. So this is a check from the
      * volume set option such that if debug xlators such as trace/errorgen
      * are provided in the set command, warn the user.
+     * volume set option such that if user custom xlators or debug
+     * xlators such as trace/errorgen are provided in the set command,
+     * warn the user.
      */
-    debug_xlator = is_server_debug_xlator(myframe);
+    ret = added_server_xlator(myframe, &added_xlator);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "failed to check that server graph has been changed");
+        goto out;
+    }
 
     if (dict_get_str(dict, "help-str", &help_str) && !msg[0])
         snprintf(msg, sizeof(msg), "Set volume %s",
                  (rsp.op_ret) ? "unsuccessful" : "successful");
-    if (rsp.op_ret == 0 && debug_xlator) {
+    if (rsp.op_ret == 0 && added_xlator) {
         snprintf(tmp_str, sizeof(tmp_str),
                  "\n%s translator has been "
                  "added to the server volume file. Please restart the"
                  " volume for enabling the translator",
-                 debug_xlator);
+                 added_xlator);
     }
 
     if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
@@ -2394,7 +2484,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
             cli_err("volume set: failed");
     } else {
         if (help_str == NULL) {
-            if (debug_xlator == NULL)
+            if (added_xlator == NULL)
                 cli_out("volume set: success");
             else
                 cli_out("volume set: success%s", tmp_str);
@@ -2408,7 +2498,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
 out:
     if (dict)
         dict_unref(dict);
-    GF_FREE(debug_xlator);
+    GF_FREE(added_xlator);
     cli_cmd_broadcast_response(ret);
     gf_free_xdr_cli_rsp(rsp);
     return ret;
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 7b4f446..b5b69ea 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -502,8 +502,6 @@ cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
 int
 cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict,
                                     char *key);
-char *
-is_server_debug_xlator(void *myframe);
 
 int32_t
 cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
diff --git a/tests/basic/user-xlator.t b/tests/basic/user-xlator.t
new file mode 100755
index 0000000..a711f9f
--- /dev/null
+++ b/tests/basic/user-xlator.t
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#### patchy.dev.d-backends-patchy1.vol
+brick=${B0//\//-}
+SERVER_VOLFILE="/var/lib/glusterd/vols/${V0}/${V0}.${H0}.${brick:1}-${V0}1.vol"
+
+cleanup;
+
+TEST mkdir -p $B0/single-brick
+TEST mkdir -p ${GLUSTER_XLATOR_DIR}/user
+
+## deploy dummy user xlator
+TEST cp ${GLUSTER_XLATOR_DIR}/playground/template.so ${GLUSTER_XLATOR_DIR}/user/hoge.so
+
+TEST glusterd
+TEST $CLI volume create $V0 replica 3  $H0:$B0/${V0}{1,2,3,4,5,6};
+TEST $CLI volume set $V0 user.xlator.hoge posix
+TEST grep -q 'user/hoge' ${SERVER_VOLFILE}
+
+TEST $CLI volume set $V0 user.xlator.hoge.opt1 10
+TEST grep -q '"option opt1 10"' ${SERVER_VOLFILE}
+TEST $CLI volume set $V0 user.xlator.hoge.opt2 hogehoge
+TEST grep -q '"option opt2 hogehoge"' ${SERVER_VOLFILE}
+TEST $CLI volume set $V0 user.xlator.hoge.opt3 true
+TEST grep -q '"option opt3 true"' ${SERVER_VOLFILE}
+
+TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6
+
+TEST $CLI volume set $V0 user.xlator.hoge trash
+TEST grep -q 'user/hoge' ${SERVER_VOLFILE}
+
+TEST $CLI volume stop $V0
+TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6
+
+TEST ! $CLI volume set $V0 user.xlator.hoge unknown
+TEST grep -q 'user/hoge' ${SERVER_VOLFILE} # When the CLI fails, the volfile is not modified.
+
+TEST $CLI volume stop $V0
+TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6
+
+#### teardown
+
+TEST rm -f ${GLUSTER_XLATOR_DIR}/user/hoge.so
+cleanup;
diff --git a/tests/env.rc.in b/tests/env.rc.in
index c7472a7..1f0ca88 100644
--- a/tests/env.rc.in
+++ b/tests/env.rc.in
@@ -40,3 +40,6 @@ export GLUSTER_LIBEXECDIR
 
 RUN_NFS_TESTS=@BUILD_GNFS@
 export RUN_NFS_TESTS
+
+GLUSTER_XLATOR_DIR=@libdir@/glusterfs/@PACKAGE_VERSION@/xlator
+export GLUSTER_XLATOR_DIR
\ No newline at end of file
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 1920284..a242b5c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -45,6 +45,11 @@ struct gd_validate_reconf_opts {
 
 extern struct volopt_map_entry glusterd_volopt_map[];
 
+struct check_and_add_user_xlator_t {
+    volgen_graph_t *graph;
+    char *volname;
+};
+
 #define RPC_SET_OPT(XL, CLI_OPT, XLATOR_OPT, ERROR_CMD)                        \
     do {                                                                       \
         char *_value = NULL;                                                   \
@@ -2822,6 +2827,145 @@ out:
     return ret;
 }
 
+static gf_boolean_t
+check_user_xlator_position(dict_t *dict, char *key, data_t *value,
+                           void *prev_xlname)
+{
+    if (strncmp(key, "user.xlator.", SLEN("user.xlator.")) != 0) {
+        return false;
+    }
+
+    if (fnmatch("user.xlator.*.*", key, 0) == 0) {
+        return false;
+    }
+
+    char *value_str = data_to_str(value);
+    if (!value_str) {
+        return false;
+    }
+
+    if (strcmp(value_str, prev_xlname) == 0) {
+        gf_log("glusterd", GF_LOG_INFO,
+               "found insert position of user-xlator(%s)", key);
+        return true;
+    }
+
+    return false;
+}
+
+static int
+set_user_xlator_option(dict_t *set_dict, char *key, data_t *value, void *data)
+{
+    xlator_t *xl = data;
+    char *optname = strrchr(key, '.') + 1;
+
+    gf_log("glusterd", GF_LOG_DEBUG, "set user xlator option %s = %s", key,
+           value->data);
+
+    return xlator_set_option(xl, optname, strlen(optname), data_to_str(value));
+}
+
+static int
+insert_user_xlator_to_graph(dict_t *set_dict, char *key, data_t *value,
+                            void *action_data)
+{
+    int ret = -1;
+
+    struct check_and_add_user_xlator_t *data = action_data;
+
+    char *xlator_name = strrchr(key, '.') + 1;  // user.xlator.<xlator_name>
+    char *xlator_option_matcher = NULL;
+    char *type = NULL;
+    xlator_t *xl = NULL;
+
+    // convert optkey to xlator type
+    if (gf_asprintf(&type, "user/%s", xlator_name) < 0) {
+        gf_log("glusterd", GF_LOG_ERROR, "failed to generate user-xlator type");
+        goto out;
+    }
+
+    gf_log("glusterd", GF_LOG_INFO, "add user xlator=%s to graph", type);
+
+    xl = volgen_graph_add(data->graph, type, data->volname);
+    if (!xl) {
+        goto out;
+    }
+
+    ret = gf_asprintf(&xlator_option_matcher, "user.xlator.%s.*", xlator_name);
+    if (ret < 0) {
+        gf_log("glusterd", GF_LOG_ERROR,
+               "failed to generate user-xlator option matcher");
+        goto out;
+    }
+
+    dict_foreach_fnmatch(set_dict, xlator_option_matcher,
+                         set_user_xlator_option, xl);
+
+out:
+    if (type)
+        GF_FREE(type);
+    if (xlator_option_matcher)
+        GF_FREE(xlator_option_matcher);
+
+    return ret;
+}
+
+static int
+validate_user_xlator_position(dict_t *this, char *key, data_t *value,
+                              void *unused)
+{
+    int ret = -1;
+    int i = 0;
+
+    if (!value)
+        goto out;
+
+    if (fnmatch("user.xlator.*.*", key, 0) == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    char *value_str = data_to_str(value);
+    if (!value_str)
+        goto out;
+
+    int num_xlators = sizeof(server_graph_table) /
+                      sizeof(server_graph_table[0]);
+    for (i = 0; i < num_xlators; i++) {
+        if (server_graph_table[i].dbg_key &&
+            strcmp(value_str, server_graph_table[i].dbg_key) == 0) {
+            ret = 0;
+            goto out;
+        }
+    }
+
+out:
+    if (ret == -1)
+        gf_log("glusterd", GF_LOG_ERROR, "invalid user xlator position %s = %s",
+               key, value->data);
+
+    return ret;
+}
+
+static int
+check_and_add_user_xl(volgen_graph_t *graph, dict_t *set_dict, char *volname,
+                      char *prev_xlname)
+{
+    if (!prev_xlname)
+        goto out;
+
+    struct check_and_add_user_xlator_t data = {.graph = graph,
+                                               .volname = volname};
+
+    if (dict_foreach_match(set_dict, check_user_xlator_position, prev_xlname,
+                           insert_user_xlator_to_graph, &data) < 0) {
+        return -1;
+    }
+
+out:
+    return 0;
+}
+
 static int
 server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                      dict_t *set_dict, void *param)
@@ -2831,6 +2975,12 @@ server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
     char *loglevel = NULL;
     int i = 0;
 
+    if (dict_foreach_fnmatch(set_dict, "user.xlator.*",
+                             validate_user_xlator_position, NULL) < 0) {
+        ret = -EINVAL;
+        goto out;
+    }
+
     i = sizeof(server_graph_table) / sizeof(server_graph_table[0]) - 1;
 
     while (i >= 0) {
@@ -2848,6 +2998,11 @@ server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
             goto out;
 
+        ret = check_and_add_user_xl(graph, set_dict, volinfo->volname,
+                                    server_graph_table[i].dbg_key);
+        if (ret)
+            goto out;
+
         i--;
     }
 
-- 
1.8.3.1