diff --git a/build/Makefile_ci.sh b/build/Makefile_ci.sh index 0fde54cef683eabb5c470627c2902cf6dc6af328..fb911b755e95991658321234bd654032c2f13322 100644 --- a/build/Makefile_ci.sh +++ b/build/Makefile_ci.sh @@ -292,6 +292,7 @@ func_pkg_run_basic() cp ${CANTIANDB_INSTALL}/Common.py ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/bin/ cp -d ${CANTIANDB_LIB}/libzeclient.so ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/lib/ cp -d ${CANTIANDB_LIB}/libzecommon.so ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/lib/ + cp -d ${CANTIANDB_LIB}/libdsslock.so ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/lib/ cp -d ${CANTIANDB_LIB}/libzeprotocol.so ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/lib/ cp -d ${PCRE_LIB_PATH}/libpcre2-8.so* ${CANTIANDB_BIN}/${RUN_PACK_DIR_NAME}/add-ons/ @@ -607,6 +608,7 @@ func_pkg_ctsql() cp -d ${CANTIANDB_LIB}/libzeclient.so ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/lib/ cp -d ${CANTIANDB_LIB}/libzecommon.so ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/lib/ cp -d ${CANTIANDB_LIB}/libzeprotocol.so ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/lib/ + cp -d ${CANTIANDB_LIB}/libdsslock.so ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/lib/ cp -d ${Z_LIB_PATH}/libz.so* ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/add-ons/ cp -d ${PCRE_LIB_PATH}/libpcre2-8.so* ${CANTIANDB_BIN}/${CTSQL_PACK_DIR_NAME}/add-ons/ diff --git a/build/build_cantian_ci.sh b/build/build_cantian_ci.sh index 0fbeb7c686bb41fd4a3b8a0d330978ddfd7a681e..6e54e371683f67203da7c9e87ff4d047444ebb0c 100644 --- a/build/build_cantian_ci.sh +++ b/build/build_cantian_ci.sh @@ -40,6 +40,10 @@ function buildCtOmPackage() { fi } +function buildDssPackage() { + sh "${CURRENT_PATH}"/build_dss.sh ${BUILD_TYPE} +} + function packageSymbol() { if [[ ${BUILD_TYPE} != "release" ]]; then return @@ -79,7 +83,7 @@ function newPackageTarget() { local pkg_real_path=${TMP_PKG_PATH}/${pkg_dir_name} rm -rf ${TMP_PKG_PATH}/* - mkdir -p ${pkg_real_path}/{action,repo,config,common,zlogicrep} + mkdir -p ${pkg_real_path}/{action,repo,config,common,zlogicrep,dss} mkdir -p ${pkg_real_path}/zlogicrep/build/Cantian_PKG/file B_VERSION=$(grep -oP '\K[^<]+' "${CTDB_CODE_PATH}"/../ProductComm_DoradoAA/CI/conf/cmc/dbstore/archive_cmc_versions.xml | sed 's/Cantian //g') # 提取B_VERSION最后一个点之后的部分 @@ -99,6 +103,7 @@ function newPackageTarget() { if [[ ${BUILD_MODE} == "single" ]]; then cp -rf "${CTDB_CODE_PATH}"/pkg/deploy/single_options/* ${pkg_real_path}/action/cantian fi + cp -arf "${CTDB_CODE_PATH}"/dss/* ${pkg_real_path}/dss/ sed -i "s/#MYSQL_PKG_PREFIX_NAME#/${mysql_pkg_name}/g" ${CTDB_CODE_PATH}/CI/script/for_mysql_official/patch.sh sed -i "s/## BUILD_TYPE ENV_TYPE ##/${build_type_upper} ${ENV_TYPE}/g" ${CTDB_CODE_PATH}/CI/script/for_mysql_official/patch.sh cp -arf "${CTDB_CODE_PATH}"/CI/script/for_mysql_official ${pkg_real_path} @@ -207,7 +212,6 @@ function prepare_path() { function prepare() { prepare_path - if [[ ${BUILD_MODE} == "multiple" ]] || [[ -z ${BUILD_MODE} ]]; then echo "compiling multiple process" if [[ ${BUILD_TYPE} == "debug" ]]; then @@ -253,5 +257,5 @@ MYSQL_BUILD_TYPE="mysql_${BUILD_TYPE}" prepare buildCtOmPackage packageTarget -newPackageTarget -#packageSymbol \ No newline at end of file +buildDssPackage +newPackageTarget \ No newline at end of file diff --git a/build/build_dss.sh b/build/build_dss.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c9f7b0923ac005759fe786a9fb78c730c321eb5 --- /dev/null +++ b/build/build_dss.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -e +ENV_TYPE=$(uname -p) +CURRENT_PATH=$(dirname $(readlink -f $0)) +CTDB_CODE_PATH="${CURRENT_PATH}"/.. +BUILD_TYPE=$1 +if [[ ! -d ${CTDB_CODE_PATH} ]];then + mkdir -p ${CTDB_CODE_PATH} +fi + +if [[ ${BUILD_TYPE} == "release" ]] || [[ x"${BUILD_TYPE}" == x"" ]];then + BUILD_TYPE="Release" +else + BUILD_TYPE="Debug" + sed -i 's/"_LOG_LEVEL": 7,/"_LOG_LEVEL": 255,/g' "${CTDB_CODE_PATH}"/pkg/deploy/action/dss/config.py +fi +echo "BUILD_TYPE:${BUILD_TYPE}" + +function download_source() { + echo "Clone source start" + if [[ x"${proxy_user}" != x"" ]];then + export http_proxy=http://${proxy_user}:${proxy_pwd}@${proxy_url} + export https_proxy=${http_proxy} + export no_proxy=127.0.0.1,.huawei.com,localhost,local,.local + fi + git clone -b master https://gitee.com/opengauss/CBB.git + git clone -b master https://gitee.com/opengauss/DSS.git + if [[ ${ENV_TYPE} == "aarch64" ]];then + wget --no-check-certificate https://opengauss.obs.cn-south-1.myhuaweicloud.com/6.0.0/binarylibs/gcc10.3/openGauss-third_party_binarylibs_openEuler_2203_arm.tar.gz + tar -zxf openGauss-third_party_binarylibs_openEuler_2203_arm.tar.gz + else + wget --no-check-certificate https://opengauss.obs.cn-south-1.myhuaweicloud.com/6.0.0/binarylibs/gcc10.3/openGauss-third_party_binarylibs_Centos7.6_x86_64.tar.gz + tar -zxf openGauss-third_party_binarylibs_Centos7.6_x86_64.tar.gz + fi + + echo "Clone source success" +} + +function build_package() { + export THIRD_PATH=${CTDB_CODE_PATH}/openGauss-third_party_binarylibs_Centos7.6_x86_64 + if [[ ${ENV_TYPE} == "aarch64" ]];then + export THIRD_PATH=${CTDB_CODE_PATH}/openGauss-third_party_binarylibs_openEuler_2203_arm + fi + export CC=${THIRD_PATH}/buildtools/gcc10.3/gcc/bin/gcc + export cc=${THIRD_PATH}/buildtools/gcc10.3/gcc/bin/gcc + export GCCFOLDER=${THIRD_PATH}/buildtools/gcc10.3 + export LD_LIBRARY_PATH=${THIRD_PATH}/buildtools/gcc10.3/gcc/lib64:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=$GCCFOLDER/gcc/lib64:$GCCFOLDER/isl/lib:$GCCFOLDER/mpc/lib/:$GCCFOLDER/mpfr/lib/:$GCCFOLDER/gmp/lib/:$LD_LIBRARY_PATH + export PATH=${THIRD_PATH}/buildtools/gcc10.3/gcc/bin:${PATH} + echo "Start to compile CBB." + cd ${CTDB_CODE_PATH}/CBB + sed -i "s/OPTION(ENABLE_EXPORT_API \"Enable hidden internal api\" OFF)/OPTION(ENABLE_EXPORT_API \"Enable hidden internal api\" ON)/g" ${CTDB_CODE_PATH}/CBB/CMakeLists.txt + sh build.sh -3rd ${THIRD_PATH} -m ${BUILD_TYPE} -t cmake + cd - + echo "Start to compile DSS." + cd ${CTDB_CODE_PATH}/DSS/build/linux/opengauss + sh build.sh -3rd ${THIRD_PATH} -m ${BUILD_TYPE} -t cmake + cd - + echo "Start to copy bin/lib source." + mkdir -p "${CTDB_CODE_PATH}"/dss/{lib,bin} + cp -arf ${CTDB_CODE_PATH}/CBB/output/bin/* "${CTDB_CODE_PATH}"/dss/bin/ + cp -arf ${CTDB_CODE_PATH}/CBB/output/lib/* "${CTDB_CODE_PATH}"/dss/lib/ + cp -arf ${CTDB_CODE_PATH}/DSS/output/bin/* "${CTDB_CODE_PATH}"/dss/bin/ + cp -arf ${CTDB_CODE_PATH}/DSS/output/lib/* "${CTDB_CODE_PATH}"/dss/lib/ + echo "end to copy bin/lib source." +} +echo "build dss start." +cd ${CTDB_CODE_PATH} +download_source +build_package +cd - +echo "build dss success." \ No newline at end of file diff --git a/common/script/cantian_service.sh b/common/script/cantian_service.sh index d091c9ac6018fb8fba2bfced381ea617fa4cb865..4fc549dac4e8fe1b0754056d5d99244fde0e7693 100644 --- a/common/script/cantian_service.sh +++ b/common/script/cantian_service.sh @@ -131,7 +131,7 @@ function startDaemon() kerberos_type=`python3 ${CURRENT_PATH}/../../action/get_config_info.py "kerberos_key"` deploy_mode=`python3 ${CURRENT_PATH}/../../action/get_config_info.py "deploy_mode"` # 当前如果为去nas模式，不启动挂载 - if [[ ! -f "${DEPLOY_MODE_DBSTORE_UNIFY_FLAG}" ]] && [[ x"${deploy_mode}" != x"dbstor" ]];then + if [[ x"${deploy_mode}" != x"dbstor" ]] && [[ x"${deploy_mode}" != x"dss" ]];then mountNfs fi local cantianPid=$(getDaemonPid) diff --git a/pkg/admin/scripts/create_cluster_database.sample.sql b/pkg/admin/scripts/create_cluster_database.sample.sql index 3a80033e4d732820c4623ce64b585fe75563c69a..ccbead290db2ba12ac2f7263f0eb30e40d0a57f1 100644 --- a/pkg/admin/scripts/create_cluster_database.sample.sql +++ b/pkg/admin/scripts/create_cluster_database.sample.sql @@ -11,11 +11,11 @@ create database clustered cantian archivelog undo tablespace datafile 'dbfiles1/undo01.dat' size 1G autoextend on next 32M, 'dbfiles1/undo02.dat' size 1G autoextend on next 32M temporary tablespace TEMPFILE 'dbfiles1/temp1_01' size 160M autoextend on next 32M, 'dbfiles1/temp1_02' size 160M autoextend on next 32M nologging undo tablespace TEMPFILE 'dbfiles1/temp2_undo_01' size 128M autoextend on next 32M - logfile ('dbfiles2/redo01.dat' size 4G, 'dbfiles2/redo02.dat' size 4G, 'dbfiles2/redo03.dat' size 4G, - 'dbfiles2/redo04.dat' size 4G, 'dbfiles2/redo05.dat' size 4G, 'dbfiles2/redo06.dat' size 4G) + logfile ('dbfiles2/redo011.dat' size 4G, 'dbfiles2/redo012.dat' size 4G, 'dbfiles2/redo013.dat' size 4G, + 'dbfiles2/redo014.dat' size 4G, 'dbfiles2/redo015.dat' size 4G, 'dbfiles2/redo016.dat' size 4G) node 1 undo tablespace datafile 'dbfiles1/undo11.dat' size 1G autoextend on next 32M, 'dbfiles1/undo12.dat' size 1G autoextend on next 32M temporary tablespace TEMPFILE 'dbfiles1/temp1_11' size 160M autoextend on next 32M, 'dbfiles1/temp1_12' size 160M autoextend on next 32M nologging undo tablespace TEMPFILE 'dbfiles1/temp2_undo_11' size 128M autoextend on next 32M - logfile ('dbfiles3/redo07.dat' size 4G, 'dbfiles3/redo08.dat' size 4G, 'dbfiles3/redo09.dat' size 4G, - 'dbfiles3/redo0a.dat' size 4G, 'dbfiles3/redo11.dat' size 4G, 'dbfiles3/redo12.dat' size 4G); + logfile ('dbfiles3/redo021.dat' size 4G, 'dbfiles3/redo022.dat' size 4G, 'dbfiles3/redo023.dat' size 4G, + 'dbfiles3/redo024.dat' size 4G, 'dbfiles3/redo025.dat' size 4G, 'dbfiles3/redo26.dat' size 4G); diff --git a/pkg/deploy/action/cantian/cantian_funclib.py b/pkg/deploy/action/cantian/cantian_funclib.py index 17a8dc8dd2f5f2b872e083d355638cc7142b952b..07c8d796b4757ea638c2ce71047033d1537fabc5 100644 --- a/pkg/deploy/action/cantian/cantian_funclib.py +++ b/pkg/deploy/action/cantian/cantian_funclib.py @@ -157,7 +157,7 @@ class DefaultConfigValue(object): "_SYS_PASSWORD": "", # input by user in command line parameter or from shell command interactively "INTERCONNECT_ADDR": "", # input by user in command line parameter "LSNR_ADDR": "127.0.0.1", - "SHARED_PATH": "+vg1", + "SHARED_PATH": "", "ARCHIVE_DEST_1": "", "MAX_ARCH_FILES_SIZE" : "300G", "PAGE_CLEAN_MODE" : "ALL", @@ -249,6 +249,13 @@ class DefaultConfigValue(object): "MES_SSL_CRT_KEY_PATH": "/opt/cantian/common/config/certificates", "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})" } + if deploy_mode == "dss": + CANTIAND_CONFIG.update({ + "CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket" + }) + CANTIAND_DBG_CONFIG.update({ + "CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket" + }) CANTIAND_CONFIG.update(MES_CONFIG) CANTIAND_DBG_CONFIG.update(MES_CONFIG) diff --git a/pkg/deploy/action/cantian/cantian_install.py b/pkg/deploy/action/cantian/cantian_install.py index ee27de08719b44ce8758279fa55ec46a8672d172..0c59d560d0d14a217b661549040f974669769c7d 100644 --- a/pkg/deploy/action/cantian/cantian_install.py +++ b/pkg/deploy/action/cantian/cantian_install.py @@ -61,6 +61,8 @@ CANTIAND_WITH_MYSQL_ST = "cantiand_with_mysql_st" # single process mode with my CANTIAND_IN_CLUSTER = "cantiand_in_cluster" CANTIAND_WITH_MYSQL_IN_CLUSTER = "cantiand_with_mysql_in_cluster" MYSQLD = "mysqld" +USE_DBSTOR = ["combined", "dbstor"] +USE_LUN = ["dss"] INSTALL_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "installdb.sh") @@ -301,19 +303,22 @@ def load_config_param(json_data): global DEPLOY_MODE DEPLOY_MODE = json_data.get("deploy_mode", "").strip() g_opts.db_type = json_data.get('db_type', '').strip() - g_opts.storage_dbstore_fs = json_data.get("storage_dbstore_fs").strip() - g_opts.storage_share_fs = json_data['storage_share_fs'].strip() + g_opts.storage_dbstore_fs = json_data.get("storage_dbstore_fs", "").strip() + g_opts.storage_share_fs = json_data.get('storage_share_fs', "").strip() g_opts.namespace = json_data.get('cluster_name', 'test1').strip() - g_opts.share_logic_ip = json_data.get('share_logic_ip', '').strip() if DEPLOY_MODE == "file" else None - g_opts.archive_logic_ip = json_data.get('archive_logic_ip', '').strip() + g_opts.share_logic_ip = json_data.get('share_logic_ip', '127.0.0.1').strip() if DEPLOY_MODE == "file" else None + g_opts.archive_logic_ip = json_data.get('archive_logic_ip', '127.0.0.1').strip() g_opts.mes_type = json_data.get("mes_type", "UC").strip() if DEPLOY_MODE == "file": g_opts.mes_type = "TCP" g_opts.mes_ssl_switch = json_data.get("mes_ssl_switch", False) - storage_archive_fs = json_data['storage_archive_fs'].strip() - g_opts.use_dbstor = DEPLOY_MODE != "file" + storage_archive_fs = json_data.get('storage_archive_fs', "").strip() + g_opts.use_dbstor = DEPLOY_MODE in USE_DBSTOR + g_opts.use_gss = DEPLOY_MODE in USE_LUN g_opts.archive_location = f"""location=/{f'mnt/dbdata/remote/archive_{storage_archive_fs}' if DEPLOY_MODE != 'dbstor' else f'{storage_archive_fs}/archive'}""" + if DEPLOY_MODE in USE_LUN: + g_opts.archive_location = "location=+vg3/archive" g_opts.dbstor_deploy_mode = DEPLOY_MODE == "dbstor" metadata_str = "metadata_" + json_data.get('storage_metadata_fs', '').strip() node_str = "node" + str(g_opts.node_id) @@ -525,7 +530,7 @@ def cantian_check_share_logic_ip_isvalid(ipname, nodeip): return False return True - if DEPLOY_MODE == "dbstor": + if DEPLOY_MODE in ["dbstor", "dss"]: return True if DEPLOY_MODE == "combined" and ipname != "archive": return True @@ -1066,6 +1071,10 @@ class Installer: self.cantiand_configs["SHARED_PATH"] = "-" self.cantiand_configs["ENABLE_DBSTOR"] = "TRUE" self.cantiand_configs["DBSTOR_NAMESPACE"] = g_opts.namespace + elif g_opts.use_gss: + self.cantiand_configs["CONTROL_FILES"] = "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + self.cantiand_configs["ENABLE_DBSTOR"] = "FALSE" + self.cantiand_configs["SHARED_PATH"] = "+vg1" else: self.cantiand_configs["ENABLE_DBSTOR"] = "FALSE" self.cantiand_configs["SHARED_PATH"] = \ @@ -1470,7 +1479,7 @@ class Installer: if platform.machine() == 'aarch64' and numa_num == 0: numa_id_str = _ans[1].strip().split('-') last_numa_id = numa_id_str[-1] - if int(last_numa_id) >= 16: + if int(last_numa_id) >= 16 and g_opts.use_dbstor: numa_str = "0-1,6-11,16-" + str(last_numa_id) numa_info += numa_str + " " numa_num += 1 @@ -2084,7 +2093,8 @@ class Installer: if not g_opts.use_dbstor: common_parameters["FILE_OPTIONS"] = "FULLDIRECTIO" - + # if g_opts.use_gss: + # common_parameters["FILE_OPTIONS"] = "ASYNCH" # 1.clean old conf self.clean_old_conf(list(common_parameters.keys()), conf_file) # 2.set new conf @@ -2710,6 +2720,9 @@ class Installer: self.cantiand_configs["SHARED_PATH"] = json_data.get('SHARED_PATH', '') self.cantiand_configs["CONTROL_FILES"] = json_data.get('CONTROL_FILES', '').strip() self.cantiand_configs["DBSTOR_NAMESPACE"] = json_data.get('DBSTOR_NAMESPACE', '').strip() + elif g_opts.use_gss: + self.cantiand_configs["CONTROL_FILES"] = "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + self.cantiand_configs["SHARED_PATH"] = "+vg1" else: self.cantiand_configs["SHARED_PATH"] = '/mnt/dbdata/remote/storage_{}/data'.format( g_opts.storage_dbstore_fs) @@ -2866,12 +2879,7 @@ class Installer: file_name = "create_database.sample.sql" if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER]: file_name = "create_cluster_database.sample.sql" - create_database_sql = os.path.join(sql_file_path, file_name) - if g_opts.use_gss: - self._sed_file("dbfiles1", "+vg1", create_database_sql) - self._sed_file("dbfiles2", "+vg2", create_database_sql) - self._sed_file("dbfiles3", "+vg3", create_database_sql) - elif g_opts.use_dbstor: + if g_opts.use_dbstor: file_name = "create_dbstor_database.sample.sql" if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER]: file_name = "create_dbstor_cluster_database.sample.sql" @@ -2899,9 +2907,14 @@ class Installer: db_data_path = os.path.join(self.data, "data").replace('/', '\/') self.set_sql_redo_size_and_num(db_data_path, create_database_sql) - self._sed_file("dbfiles1", db_data_path, create_database_sql) - self._sed_file("dbfiles2", db_data_path, create_database_sql) - self._sed_file("dbfiles3", db_data_path, create_database_sql) + if g_opts.use_gss: + self._sed_file("dbfiles1", "+vg1", create_database_sql) + self._sed_file("dbfiles2", "+vg2", create_database_sql) + self._sed_file("dbfiles3", "+vg2", create_database_sql) + else: + self._sed_file("dbfiles1", db_data_path, create_database_sql) + self._sed_file("dbfiles2", db_data_path, create_database_sql) + self._sed_file("dbfiles3", db_data_path, create_database_sql) return create_database_sql @@ -3378,7 +3391,8 @@ def check_archive_dir(): start_parameters = json.load(load_fp) if start_parameters.setdefault('db_create_status', "default") == "done": return - + if DEPLOY_MODE == "dss": + return if DEPLOY_MODE != "dbstor": cantian_check_share_logic_ip_isvalid("archive", g_opts.archive_logic_ip) archive_dir = g_opts.archive_location.split("=")[1] diff --git a/pkg/deploy/action/cantian/cantian_uninstall.py b/pkg/deploy/action/cantian/cantian_uninstall.py index 81598ad49429ff6a2214ed3a48314c7250b82bd2..948f663b7f7ee63fbd8f9474d59f8fbffd9b01d4 100644 --- a/pkg/deploy/action/cantian/cantian_uninstall.py +++ b/pkg/deploy/action/cantian/cantian_uninstall.py @@ -113,7 +113,7 @@ def cantian_check_share_logic_ip_isvalid(node_ip, deploy_mode): return False return True - if deploy_mode == "dbstor": + if deploy_mode in ["dbstor", "dss"]: return True LOGGER.info("check nfs logic ip address or domain name.") if not ping_execute("ping") and not ping_execute("ping6"): diff --git a/pkg/deploy/action/cantian/install.sh b/pkg/deploy/action/cantian/install.sh index cb2555c2c52b083241a915e12011a8e4797888a3..1a3408d8552db7c4e3c362b59293addb20778a96 100644 --- a/pkg/deploy/action/cantian/install.sh +++ b/pkg/deploy/action/cantian/install.sh @@ -65,6 +65,7 @@ function cantian_install() cp -rf ${RPM_UNPACK_PATH}/add-ons /opt/cantian/cantian/server/ cp -rf ${RPM_UNPACK_PATH}/bin /opt/cantian/cantian/server/ + rm -rf /opt/cantian/cantian/server/bin/cms cp -rf ${RPM_UNPACK_PATH}/lib /opt/cantian/cantian/server/ cp -rf ${RPM_UNPACK_PATH}/admin /opt/cantian/cantian/server/ cp -rf ${RPM_UNPACK_PATH}/cfg /opt/cantian/cantian/server/ diff --git a/pkg/deploy/action/cantian/installdb.sh b/pkg/deploy/action/cantian/installdb.sh index ab2f834a1aa1d25cded4eb20a16eb7492ff7b810..366a436d0a2265a1911ec274c3ceb9edebd9b287 100644 --- a/pkg/deploy/action/cantian/installdb.sh +++ b/pkg/deploy/action/cantian/installdb.sh @@ -54,7 +54,7 @@ function err() { function wait_node1_online() { function is_db1_online_by_cms() { - ${CTDB_HOME}/bin/cms stat -res db | grep -E "^1[[:blank:]]+db[[:blank:]]+ONLINE" + cms stat -res db | grep -E "^1[[:blank:]]+db[[:blank:]]+ONLINE" } function is_db1_online_by_query() { @@ -68,33 +68,43 @@ function wait_node1_online() { function wait_node0_online() { function is_db0_online_by_cms() { - ${CTDB_HOME}/bin/cms stat -res db | awk '{print $1, $3, $6}' | grep "0 ONLINE 1" + cms stat -res db | awk '{print $1, $3, $6}' | grep "0 ONLINE 1" } wait_for_success 5400 is_db0_online_by_cms } +function dss_reghl() { + log "start register node ${NODE_ID} by dss" + dsscmd reghl -D ${DSS_HOME} >> /dev/null 2>&1 + if [ $? != 0 ]; then err "failed to register node ${NODE_ID} by dss"; fi +} + function start_cantiand() { log "================ start cantiand ${NODE_ID} ================" + ever_started=`python3 ${CURRENT_PATH}/get_config_info.py "CANTIAN_EVER_START"` + deploy_mode=`python3 ${CURRENT_PATH}/get_config_info.py "deploy_mode"` numactl_str=" " set +e numactl --hardware if [ $? -eq 0 ]; then OS_ARCH=$(uname -i) - if [[ ${OS_ARCH} =~ "aarch64" ]]; then + if [[ ${OS_ARCH} =~ "aarch64" ]] && [[ ${deploy_mode} != "dss" ]]; then CPU_CORES_NUM=`cat /proc/cpuinfo |grep "architecture" |wc -l` CPU_CORES_NUM=$((CPU_CORES_NUM - 1)) numactl_str="numactl -C 0-1,6-11,16-"${CPU_CORES_NUM}" " fi fi set -e - ever_started=`python3 ${CURRENT_PATH}/get_config_info.py "CANTIAN_EVER_START"` if [ "${NODE_ID}" != 0 ] && [ "${ever_started}" != "True" ]; then wait_node0_online || err "timeout waiting for node0" sleep 60 fi log "Start cantiand with mode=${START_MODE}, CTDB_HOME=${CTDB_HOME}, RUN_MODE=${RUN_MODE}" - + if [ ${deploy_mode} != "dss" ]; then + dss_reghl() + fi + if [ "${RUN_MODE}" == "cantiand_with_mysql" ] || [ "${RUN_MODE}" == "cantiand_with_mysql_st" ] || [ "${RUN_MODE}" == "cantiand_with_mysql_in_cluster" ]; then if [ ! -f "${MYSQL_CONFIG_FILE}" ]; then err "Invalid mysql config file: ${MYSQL_CONFIG_FILE}" @@ -178,53 +188,15 @@ function start_cantiand() { function wait_for_node1_in_cluster() { function is_node1_joined_cluster() { - ${CTDB_HOME}/bin/cms node -list | grep -q node1 + cms node -list | grep -q node1 } wait_for_success 60 is_node1_joined_cluster } -function start_cms() { - log "=========== start cms ${NODE_ID} ================" - if [ ${NODE_ID} == 0 ]; then - if [ ${CLUSTER_SIZE} == 1 ]; then - ${CTDB_HOME}/bin/cms node -add 0 node0 127.0.0.1 ${CMS_PORT[0]} - else - for ((i = 0; i < ${CLUSTER_SIZE}; i++)); do - ${CTDB_HOME}/bin/cms node -add ${i} node${i} ${NODE_IP[$i]} ${CMS_PORT[$i]} - done - fi - - ${CTDB_HOME}/bin/cms res -add db -type db -attr "script=${CTDB_HOME}/bin/cluster.sh" - elif [ ${NODE_ID} == 1 ]; then - wait_for_node1_in_cluster - fi - - ${CTDB_HOME}/bin/cms node -list - ${CTDB_HOME}/bin/cms res -list - ${CTDB_HOME}/bin/cms server -start >> ${STATUS_LOG} 2>&1 & -} - -function prepare_cms_gcc() { - if [ "${IS_RERUN}" == 1 ]; then - return 0 - fi - - if [ "${NODE_ID}" == 0 ]; then - log "zeroing ${GCC_HOME} on node ${NODE_ID}" - dd if=/dev/zero of=${GCC_HOME} bs=1M count=1024 - ${CTDB_HOME}/bin/cms gcc -reset -f - fi -} - function install_cantiand() { start_cantiand } -function install_cms() { - prepare_cms_gcc - start_cms -} - function parse_parameter() { ARGS=$(getopt -o RSP:M:T:C: -n 'installdb.sh' -- "$@") diff --git a/pkg/deploy/action/cantian_common/get_config_info.py b/pkg/deploy/action/cantian_common/get_config_info.py new file mode 100644 index 0000000000000000000000000000000000000000..ae6993470c80e5a03bcb25c66d1ff23584073811 --- /dev/null +++ b/pkg/deploy/action/cantian_common/get_config_info.py @@ -0,0 +1,38 @@ +# -*- coding: UTF-8 -*- +import sys +import os +import json + +INSTALL_SCPRIT_DIR = os.path.dirname(os.path.abspath(__file__)) +PKG_DIR = os.path.abspath(os.path.join(INSTALL_SCPRIT_DIR, "../..")) + +CONFIG_PARAMS_FILE = os.path.join(PKG_DIR, "config", "deploy_param.json") +ENV_FILE = os.path.join(PKG_DIR, "action", "env.sh") +info = {} + +with open(CONFIG_PARAMS_FILE, encoding="utf-8") as f: + _tmp = f.read() + info = json.loads(_tmp) + + +with open(ENV_FILE, "r", encoding="utf-8") as f: + env_config = f.readlines() + + +def get_value(param): + if param == "deploy_user": + for line in env_config: + if line.startswith("cantian_user"): + return line.split("=")[1].strip("\n").strip('"') + if param == "deploy_group": + for line in env_config: + if line.startswith("cantian_group"): + return line.split("=")[1].strip("\n").strip('"') + + return info.get(param, "") + + +if __name__ == "__main__": + _param = sys.argv[1] + res = get_value(_param) + print(res) diff --git a/pkg/deploy/action/cms/cms_start2.sh b/pkg/deploy/action/cms/cms_start2.sh index 5796dd596d45c073849d59c84e9e99be064786f9..22b673df587d47a370d9ec8a1f5624229eb72cc2 100644 --- a/pkg/deploy/action/cms/cms_start2.sh +++ b/pkg/deploy/action/cms/cms_start2.sh @@ -74,6 +74,8 @@ start_cms() { log "=========== wait for cms server start ================" wait_for_cms_start + log "=========== start dss ${NODE_ID} ================" + cms res -start dss -node ${NODE_ID} log "=========== start cantian ${NODE_ID} ================" cms res -start db -node ${NODE_ID} } diff --git a/pkg/deploy/action/cms/cmsctl.py b/pkg/deploy/action/cms/cmsctl.py index 48a8644009f48bbe1db6511a6e4c10dac97ef023..be415f9c984b865f3734c17809424c5dc4001fe1 100644 --- a/pkg/deploy/action/cms/cmsctl.py +++ b/pkg/deploy/action/cms/cmsctl.py @@ -72,6 +72,8 @@ CANTIAND_WITH_MYSQL_IN_CLUSTER = "cantiand_with_mysql_in_cluster" MYSQLD = "mysqld" CANTIAN_USER = get_value("deploy_user") CANTIAN_GROUP = get_value("deploy_user") +USE_DBSTOR = ["dbstor", "combined"] +USE_DSS = ["dss"] VALID_RUNNING_MODE = {CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER, MYSQLD} @@ -248,26 +250,26 @@ def check_user(user, group): if FORCE_UNINSTALL != "force": raise Exception(err_msg) elif user == "root" or user_.pw_uid == 0: - LOGGER.error("parameter input error: -U, can not install program to" - " root user.") + LOGGER.error("parameter input error: -U, can not install program to " + "root user.") if FORCE_UNINSTALL != "force": - raise Exception("parameter input error: -U, can not install program to" - " root user.") + raise Exception("parameter input error: -U, can not install program to " + "root user.") elif group == "root" or user_.pw_gid == 0: LOGGER.error("parameter input error: -U, can not install program to" - " user with root privilege.") + " user with root privilege.") if FORCE_UNINSTALL != "force": - raise Exception("parameter input error: -U, can not install program to" - " user with root privilege.") + raise Exception("parameter input error: -U, can not install program to " + " user with root privilege.") runner_uid = os.getuid() if runner_uid != 0 and runner_uid != user_.pw_uid: runner = pwd.getpwuid(runner_uid).pw_name - LOGGER.error("Parameter input error: -U, has to be the same as the" - " executor [%s]" % runner) + LOGGER.error("Parameter input error: -U, has to be the same as the " + "executor [%s]" % runner) if FORCE_UNINSTALL != "force": raise Exception("Parameter input error: -U, has to be the same as the" - " executor [%s]" % runner) + " executor [%s]" % runner) def check_path(path_type_in): @@ -346,9 +348,10 @@ def genreg_string(text): reg_string += SEP_SED + i return reg_string + deploy_mode = get_value("deploy_mode") cantian_in_container = get_value("cantian_in_container") -mes_type = get_value("mes_type") if deploy_mode != "file" else "TCP" +mes_type = get_value("mes_type") if deploy_mode in USE_DBSTOR else "TCP" mes_ssl_switch = get_value("mes_ssl_switch") node_id = get_value("node_id") storage_share_fs = get_value("storage_share_fs") @@ -374,10 +377,10 @@ CMS_CONFIG = { "_CMS_MES_MESSAGE_QUEUE_COUNT": "1", "_CMS_MES_MESSAGE_BUFF_COUNT": "4096", "_CMS_MES_MESSAGE_CHANNEL_NUM": "1", - "_CMS_GCC_BAK": "", # generate by installer + "_CMS_GCC_BAK": "", # generate by installer "_CLUSTER_ID": 0, - "_USE_DBSTOR": "", # generate by installer - "_DBSTOR_NAMESPACE": "", # generate by installer + "_USE_DBSTOR": "", # generate by installer + "_DBSTOR_NAMESPACE": "", # generate by installer "_CMS_MES_PIPE_TYPE": mes_type } PRIMARY_KEYSTORE = "/opt/cantian/common/config/primary_keystore_bak.ks" @@ -391,6 +394,21 @@ MES_CONFIG = { "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})" } + +GCC_TYPE = { + "dbstor": "DBS", + "combined": "DBS", + "file": "file", + "dss": "SD" +} + +GCC_HOME = { + "dbstor": "", + "combined": "", + "file": "", + "dss": "" +} + CMS_CONFIG.update(MES_CONFIG) @@ -419,7 +437,7 @@ class CmsCtl(object): gcc_home = "" gcc_dir = "" - gcc_type = "NFS" if deploy_mode == "file" else "DBS" + gcc_type = GCC_TYPE.get(deploy_mode, "NFS") if os.path.exists(YOUMAI_DEMO): gcc_type = "NFS" running_mode = "cantiand_in_cluster" @@ -438,8 +456,8 @@ class CmsCtl(object): cms_home = "/opt/cantian/cms" cms_scripts = "/opt/cantian/action/cms" user_home = "" - use_gss = False - use_dbstor = True if deploy_mode != "file" else False + use_gss = deploy_mode in USE_DSS + use_dbstor = deploy_mode in USE_DBSTOR mes_type = "" cluster_uuid = "" @@ -643,9 +661,9 @@ class CmsCtl(object): config["_DBSTOR_NAMESPACE"] = self.cluster_name common_parameters = copy.deepcopy(config) - if "GCC_TYPE" in common_parameters: common_parameters["GCC_DIR"] = common_parameters["GCC_HOME"] + if "GCC_TYPE" in common_parameters and deploy_mode not in USE_DSS: common_parameters["GCC_HOME"] = os.path.join(common_parameters["GCC_HOME"], "gcc_file") self.clean_old_conf(list(common_parameters.keys()), conf_file) self.set_new_conf(common_parameters, conf_file) @@ -666,7 +684,8 @@ class CmsCtl(object): node_ip.append("127.0.0.1") gcc_home = self.gcc_home gcc_dir = self.gcc_dir - if self.gcc_type == "DBS" or self.gcc_type == "NFS": + # gcc_home is lun, gcc_home:/dev/sda + if self.gcc_type != "SD": gcc_dir = gcc_home gcc_home = os.path.join(gcc_home, "gcc_file") if 'LD_LIBRARY_PATH' in os.environ: @@ -889,7 +908,7 @@ class CmsCtl(object): LOGGER.info("change app permission cmd: %s" % str_cmd) run_cmd(str_cmd, "failed to chmod %s" % CommonValue.KEY_DIRECTORY_MODE) - if cantian_in_container == "0" and deploy_mode != "dbstor" and deploy_mode != "combined": + if cantian_in_container == "0" and (deploy_mode == "file" or os.path.exists(YOUMAI_DEMO)): self.chown_gcc_dirs() def export_user_env(self): @@ -1002,10 +1021,9 @@ class CmsCtl(object): LOGGER.info("prepare gcc home dir") self.cms_check_share_logic_ip_isvalid(self.share_logic_ip) LOGGER.info("if blocked here, please check if the network is normal") - if (deploy_mode != "dbstor" and deploy_mode != "combined" and not os.path.exists(self.gcc_home) or - os.path.exists(YOUMAI_DEMO) and not os.path.exists(self.gcc_home)): + if (deploy_mode == "file" or os.path.exists(YOUMAI_DEMO)) and not os.path.exists(self.gcc_home): os.makedirs(self.gcc_home, CommonValue.KEY_DIRECTORY_PERMISSION) - LOGGER.info("makedir for gcc_home %s" % (self.gcc_home)) + LOGGER.info("makedir for gcc_home %s" % self.gcc_home) def install_xnet_lib(self): if self.is_rdma_startup(): @@ -1123,7 +1141,7 @@ class CmsCtl(object): LOGGER.info("copy install files cmd: " + str_cmd) run_cmd(str_cmd, "failed to install cms lib files") - if deploy_mode != "file": + if deploy_mode in USE_DBSTOR: self.install_xnet_lib() self.install_kmc_lib() @@ -1144,9 +1162,12 @@ class CmsCtl(object): else: LOGGER.info("check install type : override cms") self.parse_parameters(self.install_config_file) - if deploy_mode == "dbstor" or deploy_mode == "combined": + if deploy_mode in USE_DBSTOR: self.gcc_home = os.path.join("/", self.storage_share_fs, "gcc_home") self.cms_gcc_bak = os.path.join("/", self.storage_archive_fs) + elif deploy_mode == "dss": + self.gcc_home = "/dev/gcc-disk" + self.cms_gcc_bak = "/dev/gcc-disk" else: self.gcc_home = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home") self.cms_gcc_bak = os.path.join("/mnt/dbdata/remote", "archive_" + self.storage_archive_fs) @@ -1158,7 +1179,7 @@ class CmsCtl(object): LOGGER.info("======================== begin to pre_install cms configs ========================") check_user(self.user, self.group) - if deploy_mode != "dbstor" and deploy_mode != "combined" and not check_path(self.gcc_home): + if deploy_mode == "file" and not check_path(self.gcc_home): err_msg = "the gcc home directory is invalid." LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": @@ -1234,10 +1255,10 @@ class CmsCtl(object): ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: LOGGER.error("copy dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + str(ret_code) + os.linesep + stdout + os.linesep + stderr) if FORCE_UNINSTALL != "force": raise Exception("copy dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + str(ret_code) + os.linesep + stdout + os.linesep + stderr) def install(self): """ @@ -1263,9 +1284,8 @@ class CmsCtl(object): self.set_cluster_conf() self.set_conf(CMS_CONFIG, "cms.ini") - if deploy_mode != "file": + if deploy_mode in USE_DBSTOR: self.copy_dbstor_config() - if deploy_mode == "dbstor" or deploy_mode == "combined": self.prepare_cms_tool_dbstor_config() cmd = "sh %s -P install_cms >> %s 2>&1" % (os.path.join(self.cms_scripts, "start_cms.sh"), LOG_FILE) @@ -1388,7 +1408,7 @@ class CmsCtl(object): 删除gcc_home失败时，打印出当前暂用文件的进程 :return: """ - if deploy_mode == "dbstor" or deploy_mode == "combined": + if deploy_mode in USE_DBSTOR: return gcc_home_path = "/mnt/dbdata/remote/share_%s/gcc_home/" % self.storage_share_fs remain_files = os.listdir(gcc_home_path) @@ -1409,26 +1429,33 @@ class CmsCtl(object): """ LOGGER.info("======================== begin to uninstall cms module ========================") - if deploy_mode != "dbstor" and deploy_mode != "combined" and self.gcc_home == "": + if deploy_mode not in USE_DBSTOR and self.gcc_home == "": self.gcc_home = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home") + + if deploy_mode == "dss" and self.gcc_home == "": + self.gcc_home = "/dev/gcc-disk" if self.node_id == 0: + stdout, stderr = "", "" if cantian_in_container == "0": self.cms_check_share_logic_ip_isvalid(self.share_logic_ip) LOGGER.info("if blocked here, please check if the network is normal") - if deploy_mode != "dbstor" and deploy_mode != "combined" or os.path.exists(YOUMAI_DEMO): + if deploy_mode == "file" or os.path.exists(YOUMAI_DEMO): versions_yml = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "versions.yml") gcc_backup = os.path.join("/mnt/dbdata/remote/archive_" + self.storage_archive_fs, "gcc_backup") str_cmd = "rm -rf %s && rm -rf %s && rm -rf %s" % (self.gcc_home, versions_yml, gcc_backup) ret_code, stdout, stderr = _exec_popen("timeout 10 ls %s" % self.gcc_home) - else: + if deploy_mode in USE_DBSTOR: str_cmd = "cms gcc -del && dbstor --delete-file --fs-name=%s --file-name=versions.yml && " \ "dbstor --delete-file --fs-name=%s --file-name=gcc_backup" \ % (self.storage_share_fs, self.storage_archive_fs) ret_code = 0 + if deploy_mode in USE_DSS: + str_cmd = "dd if=/dev/zero of=/dev/gcc-disk bs=1M count=1 conv=notrunc" + ret_code = 0 if ret_code == 0: LOGGER.info("clean gcc home cmd : %s" % str_cmd) ret_code, stdout, stderr = _exec_popen(str_cmd) - if ret_code and (deploy_mode == "dbstor" or deploy_mode == "combined") and self.install_step < 2: + if ret_code and deploy_mode not in USE_DBSTOR and self.install_step < 2: LOGGER.info("cms install failed, no need to clean gcc file") elif ret_code: output = stdout + stderr @@ -1443,11 +1470,13 @@ class CmsCtl(object): if FORCE_UNINSTALL != "force": raise Exception(err_msg) elif FORCE_UNINSTALL != "force" and ret_code != 2: - LOGGER.error("can not connect to remote %s" - "ret_code : %s, stdout : %s, stderr : %s" % (self.gcc_home, ret_code, stdout, stderr)) + LOGGER.error("can not connect to remote %s " + "ret_code : %s, stdout : %s, stderr : %s" % + (self.gcc_home, ret_code, stdout, stderr)) if FORCE_UNINSTALL != "force": - raise Exception("can not connect to remote %s" - "ret_code : %s, stdout : %s, stderr : %s" % (self.gcc_home, ret_code, stdout, stderr)) + raise Exception("can not connect to remote %s " + "ret_code : %s, stdout : %s, stderr : %s" % + (self.gcc_home, ret_code, stdout, stderr)) self.clean_environment() self.clean_install_path() @@ -1476,7 +1505,7 @@ class CmsCtl(object): def upgrade(self): LOGGER.info("======================== begin to upgrade cms dbstor config ========================") - if (deploy_mode == "dbstor" or deploy_mode == "combined") and not glob.glob("%s/dbstor/conf/dbs/dbstor_config_tool*" % self.cms_home): + if deploy_mode in USE_DBSTOR and not glob.glob("%s/dbstor/conf/dbs/dbstor_config_tool*" % self.cms_home): self.copy_dbstor_config() self.prepare_cms_tool_dbstor_config() @@ -1490,7 +1519,7 @@ class CmsCtl(object): if self.install_step == 3: LOGGER.info("Warning: cms start already") return - if deploy_mode != "file": + if deploy_mode in USE_DBSTOR: self.copy_dbstor_config() if deploy_mode == "dbstor" or deploy_mode == "combined": self.prepare_cms_tool_dbstor_config() diff --git a/pkg/deploy/action/cms/start_cms.sh b/pkg/deploy/action/cms/start_cms.sh index eb695ffbb0533e9d8394284f374cb7ed85474b7c..10b955ba9f8b5b64a530f530e17e61c4cf01bada 100644 --- a/pkg/deploy/action/cms/start_cms.sh +++ b/pkg/deploy/action/cms/start_cms.sh @@ -66,11 +66,15 @@ function prepare_cms_gcc() { fi if [ "${NODE_ID}" == 0 ]; then - if [[ ${DEPLOY_MODE} != "dbstor" && ${DEPLOY_MODE} != "combined" ]] || [[ -f /opt/cantian/youmai_demo ]]; then + if [[ ${DEPLOY_MODE} == "file" ]] || [[ -f /opt/cantian/youmai_demo ]]; then rm -rf ${GCC_HOME}* log "zeroing ${GCC_HOME} on node ${NODE_ID}" dd if=/dev/zero of=${GCC_HOME} bs=1M count=1024 chmod 600 ${GCC_HOME} + elif [[ ${DEPLOY_MODE} == "dss" ]]; then + log "zeroing ${GCC_HOME} on node ${NODE_ID}" + dd if=/dev/zero of=${GCC_HOME} bs=1M count=1024 + chmod 600 ${GCC_HOME} else cms gcc -create if [ $? -ne 0 ]; then @@ -172,6 +176,9 @@ function set_cms() { } function install_cms() { + if [[ ${DEPLOY_MODE} == "dss" ]];then + sudo setcap CAP_SYS_RAWIO+ep "${CMS_INSTALL_PATH}"/bin/cms + fi if [[ ${cantian_in_container} == "0" ]]; then prepare_cms_gcc set_cms diff --git a/pkg/deploy/action/config_params_lun.json b/pkg/deploy/action/config_params_lun.json new file mode 100644 index 0000000000000000000000000000000000000000..980e50011296271beacfe2d5bea88c756c073e44 --- /dev/null +++ b/pkg/deploy/action/config_params_lun.json @@ -0,0 +1,13 @@ +{ + "deploy_mode": "dss", + "deploy_user": "ctdba:ctdba", + "node_id": "0", + "cms_ip": "192.168.86.1;192.168.86.2", + "mysql_in_container": "1", + "db_type": "1", + "mes_ssl_switch": false, + "mysql_metadata_in_cantian": true, + "MAX_ARCH_FILES_SIZE": "300G", + "redo_num": "6", + "redo_size": "5G" +} diff --git a/pkg/deploy/action/dbstor/kmc_adapter.py b/pkg/deploy/action/dbstor/kmc_adapter.py index 643b3731bc4008e3ffc70d142f6d42c57137c8e4..a005f971247377b550de60788d6dd69f89ffd0c0 100644 --- a/pkg/deploy/action/dbstor/kmc_adapter.py +++ b/pkg/deploy/action/dbstor/kmc_adapter.py @@ -19,6 +19,7 @@ from enum import Enum SEC_PATH_MAX = 4096 # PATH_MAX: 待确认linux环境下大小 MAX_MK_COUNT = 4096 +USE_DBSTOR = ["combined", "dbstor"] JS_CONF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") @@ -202,7 +203,7 @@ class CApiWrapper(object): self.primary_keystore = primary_keystore self.standby_keystore = standby_keystore self.get_dbstor_para() - if self.deploy_mode != "file": + if self.deploy_mode in USE_DBSTOR: self.kmc_ext = ctypes.cdll.LoadLibrary(self.KMC_EXT_LIB_PATH) def get_dbstor_para(self): @@ -211,7 +212,7 @@ class CApiWrapper(object): self.deploy_mode = json_data.get('deploy_mode', "").strip() def initialize(self): - if self.initialized or self.deploy_mode == "file": + if self.initialized or self.deploy_mode not in USE_DBSTOR: return 0 conf = KmcInitConf( @@ -282,9 +283,9 @@ class CApiWrapper(object): return encoded.decode('utf-8') def encrypt(self, plain): - if self.deploy_mode != "file": + if self.deploy_mode in USE_DBSTOR: return self.encrypt_by_kmc(plain) - elif self.deploy_mode == "file": + elif self.deploy_mode not in USE_DBSTOR: return self.encrypt_by_base64(plain) return "" @@ -311,14 +312,14 @@ class CApiWrapper(object): return decoded.decode('utf-8') def decrypt(self, cipher): - if self.deploy_mode != "file": + if self.deploy_mode in USE_DBSTOR: return self.decrypt_by_kmc(cipher) - elif self.deploy_mode == "file": + elif self.deploy_mode not in USE_DBSTOR: return self.decrypt_by_base64(cipher) return "" def finalize(self): - if not self.initialized or self.deploy_mode == "file": + if not self.initialized or self.deploy_mode not in USE_DBSTOR: return 0 return self.kmc_ext.KeFinalizeEx(pointer(self.kmc_ctx)) diff --git a/pkg/deploy/action/dss/appctl.sh b/pkg/deploy/action/dss/appctl.sh new file mode 100644 index 0000000000000000000000000000000000000000..ef9ba487e2cc216516c5bb614ef781ca828b8edc --- /dev/null +++ b/pkg/deploy/action/dss/appctl.sh @@ -0,0 +1,153 @@ +#!/bin/bash +################################################################################ +# 【功能说明】 +# 1.appctl.sh由管控面调用 +# 2.完成如下流程 +# 服务初次安装顺序:pre_install->install->start->check_status +# 服务带配置安装顺序:pre_install->install->restore->start->check_status +# 服务卸载顺序:stop->uninstall +# 服务带配置卸载顺序:backup->stop->uninstall +# 服务重启顺序:stop->start->check_status + +# 服务A升级到B版本:pre_upgrade(B)->stop(A)->update(B)->start(B)->check_status(B) +# update(B)=备份A数据，调用A脚本卸载，调用B脚本安装，恢复A数据(升级数据) +# 服务B回滚到A版本:stop(B)->rollback(B)->start(A)->check_status(A)->online(A) +# rollback(B)=根据升级失败标志调用A或是B的卸载脚本，调用A脚本安装，数据回滚特殊处理 +# 3.典型流程路线：install(A)-upgrade(B)-upgrade(C)-rollback(B)-upgrade(C)-uninstall(C) +# 【返回】 +# 0：成功 +# 1：失败 +# +# 【注意事项】 +# 1.所有的操作需要支持失败可重入 +################################################################################ +set +x +#当前路径 +CURRENT_PATH=$(dirname $(readlink -f "$0")) + +#脚本名称 +PARENT_DIR_NAME=$(pwd | awk -F "/" '{print $NF}') +SCRIPT_NAME=${PARENT_DIR_NAME}/$(basename "$0") +DSS_SOURCE=${CURRENT_PATH}/../../dss +dss_scripts=/opt/cantian/action/dss + +#依赖文件 +source "${CURRENT_PATH}"/../log4sh.sh +source "${CURRENT_PATH}"/../env.sh + +function usage() +{ + echo "Usage: ${0##*/} {start|startup|shutdown|stop|install|uninstall|pre_upgrade| + upgrade_backup|upgrade|rollback|check_status|init_container}. [Line:${LINENO}, File:${SCRIPT_NAME}]" + return 1 +} + +function log() { + printf "[%s] %s\n" "`date -d today \"+%Y-%m-%d %H:%M:%S.%N\"`" "$1" +} + +# 通用入口 +function do_deploy() +{ + local action=$1 + local mode=$2 + su -s /bin/bash - "${cantian_user}" -c "python3 -B ${CURRENT_PATH}/dssctl.py ${action} ${mode}" + ret=$? + if [ $ret -ne 0 ]; then + echo "Execute ${COMPONENTNAME} dssctl.py return ${ret}. [Line:${LINENO}, File:${SCRIPT_NAME}]" + return 1 + fi + return 0 +} + +function permission_opt() { + chmod 500 "${DSS_SOURCE}"/bin/* + chown -hR "${cantian_user}":"${cantian_group}" "${DSS_SOURCE}" + chown "${cantian_user}":"${cantian_group}" "${CURRENT_PATH}"/* + chown root:root "${CURRENT_PATH}"/appctl.sh + mkdir -p /opt/cantian/dss/log + touch /opt/cantian/dss/log/dss_deploy.log + chown -hR "${cantian_user}":"${cantian_group}" /opt/cantian/dss/ +} + +function copy_dss_scripts() +{ + echo "copying the cms scripts" + clean_dss_scripts + mkdir -p ${dss_scripts} + chmod 755 ${dss_scripts} + cp -arf ${CURRENT_PATH}/* ${dss_scripts}/ +} + +function clean_dss_scripts() +{ + if [ -d ${dss_scripts} ]; then + rm -rf ${dss_scripts} + fi +} + +ACTION=$1 +if [ $# -gt 1 ]; then + BACKUP_UPGRADE_PATH=$2 + BIN_PATH=$2 + START_MODE=$2 +fi +if [ $# -gt 2 ]; then + BACKUP_UPGRADE_PATH=$3 +fi + + +function main() +{ + case "$ACTION" in + start) + do_deploy "--action=${ACTION}" + exit $? + ;; + stop) + do_deploy "--action=${ACTION}" + exit $? + ;; + pre_install) + permission_opt + do_deploy "--action=${ACTION}" + exit $? + ;; + install) + permission_opt + copy_dss_scripts + do_deploy "--action=${ACTION} --mode=${START_MODE}" + exit $? + ;; + uninstall) + permission_opt + do_deploy "--action=${ACTION} --mode=${START_MODE}" + exit $? + ;; + pre_upgrade) + do_deploy "--action=${ACTION}" + exit $? + ;; + upgrade_backup) + do_deploy "--action=${ACTION}" + exit $? + ;; + upgrade) + do_deploy "--action=${ACTION}" + exit $? + ;; + rollback) + do_deploy "--action=${ACTION}" + exit $? + ;; + check_status) + do_deploy "--action=${ACTION}" + exit $? + ;; + *) + usage + exit $? + ;; + esac +} +main diff --git a/pkg/deploy/action/dss/config.py b/pkg/deploy/action/dss/config.py new file mode 100644 index 0000000000000000000000000000000000000000..772030077ec5c61fbda1662192af831888ac798d --- /dev/null +++ b/pkg/deploy/action/dss/config.py @@ -0,0 +1,16 @@ +VG_CONFIG = { + "vg1": "/dev/dss-disk1", + "vg2": "/dev/dss-disk2", + "vg3": "/dev/dss-disk3" +} + +INST_CONFIG = { + "INST_ID": 0, + "_LOG_LEVEL": 7, + "STORAGE_MODE": "CLUSTER_RAID", + "DSS_NODES_LIST": "0:127.0.0.1:1811, 1:127.0.0.2:1811", + "LSNR_PATH": "", + "_LOG_BACKUP_FILE_COUNT": "40", + "_LOG_MAX_FILE_SIZE": "120M", + "DSS_CM_SO_NAME": "libdsslock.so" +} diff --git a/pkg/deploy/action/dss/dss_contrl.sh b/pkg/deploy/action/dss/dss_contrl.sh new file mode 100644 index 0000000000000000000000000000000000000000..64673763bdffe4686873f4597b0843d93c2d12b6 --- /dev/null +++ b/pkg/deploy/action/dss/dss_contrl.sh @@ -0,0 +1,279 @@ +#!/bin/bash +source ~/.bashrc + +dbuser=`whoami` +loguser=`whoami` +if [ "${dbuser}" = "root" ] +then + dbuser=$(grep '"U_USERNAME_AND_GROUP"' /opt/cantian/action/cantian/install_config.json | cut -d '"' -f 4 | sed 's/:.*//') +fi + +DSS_BIN=dssserver +BIN_PATH=${DSS_HOME}/bin +SCRIPT_NAME=`basename $(readlink -f $0)` +dss_config="${DSS_HOME}/cfg" +CONN_PATH=UDS:${DSS_HOME}/.dss_unix_d_socket +parm=$1 +node_id=$2 + +usage() +{ + echo "Usage:" + echo "cmd:" + echo " $0 -start node_id: start dssserver" + echo " $0 -stop node_id: stop dssserver&create dn_stop_flag_file" + echo " $0 -check node_id: check dssserver" + echo " $0 -clean node_id: clean dssserver" + echo " $0 -reg node_id: register dssserver" + echo " $0 -kick node_id: unregister dssserver" + echo " $0 -isreg node_id: check whether dssserver is registered" +} + +program_pid() +{ + pid=`ps -f f -u ${dbuser} | grep -w ${DSS_BIN} | grep ${DSS_HOME} | grep -v grep | grep -v ${SCRIPT_NAME} | awk '{print $2}' | tail -1` + echo ${pid} +} + +program_status() +{ + pid=`program_pid` + if [[ -z ${pid} ]]; then + echo "" + return + fi + + pstatus_file="/proc/"${pid}"/status" + cat ${pstatus_file} | while read line + do + if [[ "${line}" =~ ^State.* ]]; then + echo ${line} | awk -F ":" '{print $2}' | awk -F " " '{print $1}' + return + fi + done + + echo "" +} + +function check_dss() +{ + dss_status=$(program_status) + if [[ -z ${dss_status} ]] + then + echo "RES_FAILED" + return 1 + fi + if [[ "${dss_status}" == "D" || "${dss_status}" == "T" || "${dss_status}" == "Z" ]] + then + echo "RES_EAGAIN" + return 3 + fi + return 0 +} + +function check_dss_config() +{ + if [[ ! -e ${dss_config}/dss_inst.ini ]] + then + echo "${dss_config}/dss_inst.ini NOT exist" + echo "RES_FAILED" + exit 1 + fi + + if [[ ! -e ${DSS_HOME}/cfg/dss_vg_conf.ini ]] + then + echo "${dss_config}/dss_vg_conf.ini NOT exist" + echo "RES_FAILED" + exit 1 + fi + + LSNR_PATH=`cat ${DSS_HOME}/cfg/dss_inst.ini | sed s/[[:space:]]//g |grep -Eo "^LSNR_PATH=.*" | awk -F '=' '{print $2}'` + if [[ -z ${LSNR_PATH} ]] + then + echo "CANNOT find lsnr path." + echo "RES_FAILED" + exit 1 + fi + CONN_PATH=UDS:${LSNR_PATH}/.dss_unix_d_socket +} + +function ScandCheck() +{ + groups=`groups` + echo $groups + array=(${groups// / }) + for var in ${array[@]} + do + echo $var + nohup dsscmd scandisk -t block -p /dev/sd -u $dbuser -g $var >> /dev/null 2>&1 & + if [[ $? != 0 ]] + then + exit 1 + fi + done +} + +function start_dss() +{ + check_dss_config + ScandCheck + nohup ${BIN_PATH}/${DSS_BIN} -D ${DSS_HOME} >> /dev/null 2>&1 & +} + +function stop_dss() { + res_count=`ps -u ${dbuser} | grep ${DSS_BIN} |grep -v grep |wc -l` + echo "res_count = ${res_count}" + if [ "$res_count" -eq "0" ]; then + echo "RES_FAILED" + exit 1 + elif [ "$res_count" -eq "1" ]; then + ps -u ${dbuser} | grep ${DSS_BIN}|grep -v grep | awk '{print "kill -9 " $1}' |sh + echo "RES_SUCCESS" + exit 0 + else + res_count=`ps -fu ${dbuser} | grep ${DSS_BIN} | grep ${process_path} | grep -v grep | wc -l` + echo "res_count is ${res_count}" + if [ "$res_count" -eq "0" ]; then + echo "RES_FAILED" + exit 1 + elif [ "$res_count" -eq "1" ]; then + ps -fu ${dbuser} | grep ${DSS_BIN} | grep ${process_path} | grep -v grep | awk '{print "kill -9 " $2}' |sh + echo "RES_SUCCESS" + exit 0 + else + echo "RES_EAGAIN" + exit 3 + fi + fi +} + +function stop_dss_by_force() { + res_count=`ps -u ${dbuser} | grep ${DSS_BIN}|grep -v grep |wc -l` + echo "res_count = ${res_count}" + if [ "$res_count" -eq "0" ]; then + echo "RES_SUCCESS" + exit 0 + elif [ "$res_count" -eq "1" ]; then + ps -u ${dbuser} | grep ${DSS_BIN}|grep -v grep | awk '{print "kill -9 " $1}' |sh + echo "RES_SUCCESS" + exit 0 + else + res_count=`ps -fu ${dbuser} | grep ${DSS_BIN} | grep ${process_path} | grep -v grep | wc -l` + echo "res_count is ${res_count}" + if [ "$res_count" -eq "0" ]; then + echo "RES_SUCCESS" + exit 0 + elif [ "$res_count" -eq "1" ]; then + ps -fu ${dbuser} | grep ${DSS_BIN} | grep ${process_path} | grep -v grep | awk '{print "kill -9 " $2}' |sh + echo "RES_SUCCESS" + exit 0 + else + echo "RES_FAILED" + exit 1 + fi + fi +} + +function reg() +{ + ScandCheck + LOCAL_INSTANCE_ID=`awk '/INST_ID/{print}' ${DSS_HOME}/cfg/dss_inst.ini | awk -F= '{print $2}' | xargs` + if [[ -z ${LOCAL_INSTANCE_ID} ]] + then + echo "RES_FAILED" + exit 1 + fi + dsscmd reghl -D ${DSS_HOME} >> /dev/null 2>&1 + if [[ $? != 0 ]] + then + echo "RES_FAILED" + exit 1 + fi + echo "RES_SUCCESS" + exit 0 +} + +function kick() +{ + LOCAL_INSTANCE_ID=`awk '/INST_ID/{print}' ${DSS_HOME}/cfg/dss_inst.ini | awk -F= '{print $2}' | xargs` + if [[ -z ${LOCAL_INSTANCE_ID} ]] + then + echo "RES_FAILED" + exit 1 + fi + if [[ ${LOCAL_INSTANCE_ID} == ${node_id} ]] + then + dsscmd unreghl -D ${DSS_HOME} >> /dev/null 2>&1 + else + dsscmd kickh -i ${node_id} -D ${DSS_HOME} >> /dev/null 2>&1 + fi + + if [[ $? != 0 ]] + then + echo "RES_FAILED" + exit 1 + fi + echo "RES_SUCCESS" + exit 0 +} + +function isreg() +{ + dsscmd inq_reg -i ${node_id} -D ${DSS_HOME} >> /dev/null 2>&1 + result=$? + if [[ ${result} == 255 ]] + then + echo "RES_EAGAIN" + exit 3 + fi + if [[ ${result} != 2 ]] + then + echo "RES_FAILED" + exit 1 + fi + echo "RES_SUCCESS" + exit 0 +} + +############################### main ############################### + +if [ $# -ne 2 ]; then + usage + exit 1 +fi + +case "${parm}" in + -start) + start_dss + ;; + -stop) + stop_dss + ;; + -stop_force) + stop_dss_by_force + ;; + -check) + check_dss + if [ $? -ne 0 ]; then + echo "RES_FAILED" + exit 1 + fi + ;; + -reg) + reg + ;; + -kick) + kick + ;; + -isreg) + isreg + ;; + *) + echo "RES_FAILED" + usage + exit 1 + ;; +esac + +echo "RES_SUCCESS" +exit 0 \ No newline at end of file diff --git a/pkg/deploy/action/dss/dssctl.py b/pkg/deploy/action/dss/dssctl.py new file mode 100644 index 0000000000000000000000000000000000000000..714280af113dc07b81cf4011c46d12e378fb7c9f --- /dev/null +++ b/pkg/deploy/action/dss/dssctl.py @@ -0,0 +1,451 @@ +import argparse +import datetime +import logging +import os.path +import re +import subprocess +import shutil +import stat +import time +from logging import handlers +import sys +from config import INST_CONFIG, VG_CONFIG +from pathlib import Path + +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +from cantian_common.exec_sql import exec_popen +from cantian_common.get_config_info import get_value + + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +SOURCE_DIR = os.path.join(Path(CURRENT_DIR).parent.parent, "dss") +LOG_FILE = r"/opt/cantian/dss/log/dss_deploy.log" +DSS_HOME = "/opt/cantian/dss" +CMS_HOME = "/opt/cantian/cms/service" +DSS_CFG = "/opt/cantian/dss/cfg" +BACKUP_NAME = "/opt/cantian/backup/files/dss" +SCRIPTS_DIR = "/opt/cantian/action/dss" +DSS_CTRL_SCRIPTS = "%s/dss_contrl.sh" % SCRIPTS_DIR +RETRY_TIMES = 20 + + +# 日志组件 +def setup(): + console = logging.StreamHandler() + console.setLevel(logging.INFO) + + log = logging.getLogger("dss") + for handler in list(log.handlers): + log.removeHandler(handler) + file_log = handlers.RotatingFileHandler( + LOG_FILE, maxBytes=6291456, + backupCount=5) + log.addHandler(file_log) + log.addHandler(console) + + for handler in log.handlers: + handler.setFormatter( + logging.Formatter( + fmt="%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s]" + " [tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S")) + log.setLevel(logging.INFO) + return log + + +LOG = setup() + + +class ComOpt: + @staticmethod + def write_ini(file_path: str, contents: dict, split="=") -> None: + content = [] + for key in contents.keys(): + content.append("{}{}{}".format(key, split, contents[key])) + modes = stat.S_IWRITE | stat.S_IRUSR + flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT + with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + file.write("\n".join(content)) + os.chmod(file_path, 0o640) + + @staticmethod + def read_ini(file_path: str) -> str: + with open(file_path, 'r', encoding="utf-8") as file: + return file.read() + + +class DssCtl(object): + def __init__(self): + self.dss_inst_cfg = os.path.join(DSS_CFG, "dss_inst.ini") + self.dss_vg_cfg = os.path.join(DSS_CFG, "dss_vg_conf.ini") + self.node_id = get_value("node_id") + self.cms_ip = get_value("cms_ip") + self.mes_ssl_switch = get_value("mes_ssl_switch") + self.ca_path = get_value("ca_path") + self.crt_path = get_value("crt_path") + self.key_path = get_value("key_path") + self.log_file = os.path.join(DSS_HOME, "log/run/dssinstance.rlog") + self.begin_time = None + + @staticmethod + def modify_env(action="add") -> None: + """ + modify user environment variables + exp: DSS_HOME + LD_LIBRARY_PATH + PATH + :param action: add/delete + :return: + """ + home_directory = os.path.expanduser('~') + bashrc_path = os.path.join(home_directory, '.bashrc') + with open(bashrc_path, 'r') as bashrc_file: + bashrc_content = bashrc_file.readlines() + env = [ + "export DSS_HOME=%s\n" % DSS_HOME, + "export LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n" % os.path.join(DSS_HOME, "lib"), + "export PATH=%s:$PATH\n" % os.path.join(DSS_HOME, "bin") + ] + for line in env: + if action == "add": + if line not in bashrc_content: + bashrc_content.append(line) + if action == "delete": + if line in bashrc_content: + bashrc_content.remove(line) + + modes = stat.S_IWRITE | stat.S_IRUSR + flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT + with os.fdopen(os.open(bashrc_path, flags, modes), 'w', encoding='utf-8') as bashrc_file: + bashrc_file.writelines(bashrc_content) + + @staticmethod + def kill_cmd(cmd: str) -> None: + return_code, stdout, stderr = exec_popen(cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Dssserver is offline: %s" % str(output) + LOG.info(err_msg) + if stdout: + LOG.info("dss server pid is[%s].", stdout) + for line in re.split(r'\n\s', stdout): + kill_cmd = "kill -9 %s" % line.strip() + return_code, stdout, stderr = exec_popen(kill_cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Exec kill cmd[%s] failed, details: %s" % (cmd, str(output)) + LOG.error(err_msg) + + def wait_dss_instance_started(self): + LOG.info("Waiting for dss_instance to start...") + timeout = 60 + while timeout: + time.sleep(5) + timeout = timeout - 5 + if os.path.exists(self.log_file): + with open(self.log_file, 'r', errors='ignore') as f: + all_the_text = f.read() + succ_pattern = re.compile(r'.*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?DSS SERVER STARTED.*?', + re.IGNORECASE) + fail_pattern = re.compile(r'.*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?dss failed to startup.*?', + re.IGNORECASE) + succ_timestamps = re.findall(succ_pattern, all_the_text) + fail_timestamps = re.findall(fail_pattern, all_the_text) + is_instance_started = len(succ_timestamps) != 0 and max(succ_timestamps) >= self.begin_time + is_instance_failed = len(fail_timestamps) != 0 and max(fail_timestamps) >= self.begin_time + if is_instance_started: + LOG.info("DSS server started successfully.") + return + if is_instance_failed: + err_msg = "DSS server start failed." + LOG.error(err_msg) + raise Exception(err_msg) + else: + err_msg = "Start dss server timeout" + LOG.error(err_msg) + raise Exception(err_msg) + + def dss_cmd_add_vg(self) -> None: + """ + create volume by dsscmd. + :return: + """ + if self.node_id == "0": + LOG.info("Start to exec dsscmd cv.") + dsscmd = "source ~/.bashrc && dsscmd cv -g %s -v %s" + for key, value in VG_CONFIG.items(): + return_code, stdout, stderr = exec_popen(dsscmd % (key, value), timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Dsscmd cv cmd[%s] exec failed, details: %s" % (dsscmd % (key, value), str(output)) + raise Exception(err_msg) + LOG.info("Success to exec dsscmd cv.") + else: + LOG.info("No need to exec dsscmd cv for node[%s].", self.node_id) + + def prepare_dss_dick(self) -> None: + """ + Initialization disk + :return: + """ + if self.node_id == "0": + LOG.info("start to init lun.") + init_cmd = "dd if=/dev/zero of=%s bs=1M count=1 conv=notrunc" + for key, value in VG_CONFIG.items(): + return_code, stdout, stderr = exec_popen(init_cmd % value, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Init lun cmd[%s] exec failed, details: %s" % (init_cmd % value, str(output)) + raise Exception(err_msg) + LOG.info("Init lun cmd[] exec success.", init_cmd) + LOG.info("Success to init lun.") + else: + LOG.info("No need to init lun for node[%s].", self.node_id) + + def prepare_cfg(self) -> None: + """ + prepare dss config: dss_vg.ini/dss_inst.ini + :return: + """ + if not os.path.exists(DSS_HOME): + os.makedirs(DSS_HOME, exist_ok=True) + if not os.path.exists(DSS_CFG): + os.makedirs(DSS_CFG, exist_ok=True) + ComOpt.write_ini(self.dss_vg_cfg, VG_CONFIG, split=":") + INST_CONFIG["INST_ID"] = self.node_id + INST_CONFIG["DSS_NODES_LIST"] = "0:{}:1811,1:{}:1811".format(self.cms_ip.split(";")[0], self.cms_ip.split(";")[1]) + INST_CONFIG["LSNR_PATH"] = DSS_HOME + INST_CONFIG["LOG_HOME"] = os.path.join(DSS_HOME, "log") + ComOpt.write_ini(self.dss_inst_cfg, INST_CONFIG) + + def prepare_source(self) -> None: + """ + prepare dsscmd/dssserver lib and bin + :return: + """ + LOG.info("Copy bin and lib source start.") + if os.path.exists(os.path.join(DSS_HOME, "bin")): + shutil.rmtree(os.path.join(DSS_HOME, "bin")) + if os.path.exists(os.path.join(DSS_HOME, "lib")): + shutil.rmtree(os.path.join(DSS_HOME, "lib")) + shutil.copytree(os.path.join(SOURCE_DIR, "bin"), os.path.join(DSS_HOME, "bin")) + shutil.copytree(os.path.join(SOURCE_DIR, "lib"), os.path.join(DSS_HOME, "lib")) + LOG.info("Copy bin and lib source success.") + + def cms_add_dss_res(self) -> None: + """ + add dss res for cms + :return: + """ + if self.node_id == "0": + LOG.info("Start to add dss res.") + if not os.path.exists(DSS_CTRL_SCRIPTS): + raise Exception(f"The script file {DSS_CTRL_SCRIPTS} does not exist.") + os.chmod(DSS_CTRL_SCRIPTS, 0o550) + cmd = ("source ~/.bashrc && %s/bin/cms res -add dss -type dss -attr \"script=%s\"" + % (CMS_HOME, DSS_CTRL_SCRIPTS)) + return_code, stdout, stderr = exec_popen(cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Failed to add dss res, details: %s" % (str(output)) + raise Exception(err_msg) + LOG.info("Success to add dss res.") + + def config_perctrl_permission(self) -> None: + """ + config perctl permission for caw write. + :return: + """ + sudo_cmds = ["RAWIO", "ADMIN"] + LOG.info("Start to config perctrl permission.") + for _cmd in sudo_cmds: + exec_cmd = "sudo setcap CAP_SYS_%s+ep %s/bin/perctrl" % (_cmd, DSS_HOME) + return_code, stdout, stderr = exec_popen(exec_cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Failed to config perctl permission, details: %s" % (str(output)) + raise Exception(err_msg) + LOG.info("Success to config perctrl permission.") + + def check_is_reg(self) -> bool: + """ + check current node is reg. + :return: + """ + kick_cmd = "source ~/.bashrc && %s/bin/dsscmd inq_reg -i %s -D %s" % (DSS_HOME, self.node_id, DSS_HOME) + return_code, stdout, stderr = exec_popen(kick_cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Failed to inq_reg disk, details: %s" % (str(output)) + LOG.error(err_msg) + return "is registered" in str(stdout) + + def kick_node(self): + """ + kick node before reg. + :return: + """ + LOG.info("Start to kick node.") + kick_cmd = "source ~/.bashrc && %s/bin/dsscmd unreghl -D %s" % (DSS_HOME, DSS_HOME) + return_code, stdout, stderr = exec_popen(kick_cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Kick node cmd[%s] exec failed, details:%s" % (kick_cmd, output) + raise Exception(err_msg) + LOG.info("Success to kick node.") + + def reghl_dss_disk(self) -> None: + """ + reg current by dsscmd, dssserver is offline + :return: + """ + LOG.info("Start to reghl disk.") + if self.check_is_reg(): + self.kick_node() + reg_cmd = "source ~/.bashrc && %s/bin/dsscmd reghl -D %s" % (DSS_HOME, DSS_HOME) + return_code, stdout, stderr = exec_popen(reg_cmd, timeout=60) + if return_code: + output = stdout + stderr + err_msg = "Reghl node cmd[%s] exec failed, details:%s" % (reg_cmd, output) + raise Exception(err_msg) + LOG.info("Success to reghl disk.") + + def clean_soft(self): + """ + clean soft, bin/lib/cfg, keep logs + :return: + """ + LOG.info("Start to clean soft.") + if os.path.exists(os.path.join(DSS_HOME, "lib")): + shutil.rmtree(os.path.join(DSS_HOME, "lib")) + if os.path.exists(os.path.join(DSS_HOME, "bin")): + shutil.rmtree(os.path.join(DSS_HOME, "bin")) + if os.path.exists(os.path.join(DSS_HOME, "cfg")): + shutil.rmtree(os.path.join(DSS_HOME, "cfg")) + LOG.info("Success to clean soft.") + + def pre_install(self, *args) -> None: + """ + pre-check before install + :param args: + :return: + """ + LOG.info("Start to pre install.") + LOG.info("Success to pre install.") + + def install(self, *args) -> None: + """ + install dss: + add user environment variables; + prepare dss lib and bin source; + add res; + reg current node to dssserver ; + init disk; + create vg by dsscmd; + :param args: + :return: + """ + self.modify_env(action="add") + self.prepare_cfg() + self.prepare_source() + self.cms_add_dss_res() + self.config_perctrl_permission() + self.prepare_dss_dick() + self.reghl_dss_disk() + self.dss_cmd_add_vg() + + def backup(self, *args) -> None: + LOG.info("Start backup.") + if not os.path.exists(BACKUP_NAME): + os.makedirs(BACKUP_NAME, exist_ok=True) + shutil.copytree(DSS_CFG, BACKUP_NAME, + symlinks=False, ignore=None, + copy_function=shutil.copy2, + ignore_dangling_symlinks=False) + if not os.path.exists(os.path.join(BACKUP_NAME, "scripts")): + os.makedirs(os.path.join(BACKUP_NAME, "scripts")) + shutil.copytree(SCRIPTS_DIR, os.path.join(BACKUP_NAME, "scripts")) + LOG.info("Success to backup.") + + def uninstall(self, *args) -> None: + """ + uninstall dss: + - clean user environment variables; + - clean soft. + :param args: + :return: + """ + self.modify_env(action="delete") + self.clean_soft() + + def start(self, *args) -> None: + """ + start dss server: + - check dss server status + - start dss server + :param args: + :return: + """ + LOG.info("Start dss server.") + if self.check_status(): + return + os.chmod(DSS_CTRL_SCRIPTS, 0o550) + self.begin_time = str(datetime.datetime.now()).split(".")[0] + dssserver_cmd = "source ~/.bashrc && cms res -start dss -node %s" % self.node_id + subprocess.Popen(dssserver_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.wait_dss_instance_started() + if self.check_status(): + LOG.info("Success to start dss server.") + else: + err_msg = "Failed to start dss server." + LOG.error(err_msg) + raise Exception(err_msg) + + def stop(self, *args) -> None: + """ + 1、stop dssserver + 2、stop perctrl + :param args: + :return: + """ + LOG.info("Start to stop dss server.") + dss_server_pid = "pidof dssserver" + self.kill_cmd(dss_server_pid) + LOG.info("Success to stop dss server.") + LOG.info("Start to stop perctrl.") + perctrl_pid = "ps -ef | grep perctrl | grep -v grep | awk '{print $2}'" + self.kill_cmd(perctrl_pid) + LOG.info("Success to stop perctrl.") + + def check_status(self, *args) -> bool: + LOG.info("Start check status start") + check_cmd = "ps -ef | grep dssserver | grep -v grep | grep %s" % DSS_HOME + _, stdout, stderr = exec_popen(check_cmd, timeout=60) + if stdout: + LOG.info("dssserver is online, status: %s" % stdout) + return True + else: + LOG.info("dssserver is offline.") + return False + + +def main(): + parse = argparse.ArgumentParser() + parse.add_argument("--action", type=str, + choices=["install", "uninstall", "start", "stop", "pre_install", + "upgrade", "rollback", "pre_upgrade"]) + parse.add_argument("--mode", required=False, dest="mode", default="") + arg = parse.parse_args() + act = arg.action + dss_opt = DssCtl() + getattr(dss_opt, act)(arg.mode) + + +if __name__ == "__main__": + try: + main() + except Exception as err: + LOG.error(str(err)) + exit(str(err)) + exit(0) diff --git a/pkg/deploy/action/env.sh b/pkg/deploy/action/env.sh index 1b9d9f13ba9b6d0e880d6c4f9d40b4697f7079da..a580a453a674e8f88c29952fc30bc0886272d18a 100644 --- a/pkg/deploy/action/env.sh +++ b/pkg/deploy/action/env.sh @@ -9,16 +9,16 @@ cantian_user="cantian" cantian_group="cantian" cantian_common_group="cantiangroup" -PRE_INSTALL_ORDER=("cantian" "cms" "dbstor") -INSTALL_ORDER=("dbstor" "cms" "cantian" "ct_om" "cantian_exporter" "mysql" "logicrep") -START_ORDER=("cms" "cantian" "ct_om" "cantian_exporter" "logicrep") -STOP_ORDER=("logicrep" "cms" "cantian" "ct_om" "cantian_exporter") -UNINSTALL_ORDER=("ct_om" "cantian" "cms" "dbstor" "logicrep") -BACKUP_ORDER=("cantian" "cms" "ct_om" "dbstor" ) -CHECK_STATUS=("cantian" "cms" "ct_om" "cantian_exporter") -PRE_UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "cantian" "mysql" "logicrep") -UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "cantian" "mysql" "logicrep") -POST_UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "cantian" "mysql") +PRE_INSTALL_ORDER=("cantian" "cms" "dbstor" "dss") +INSTALL_ORDER=("dbstor" "cms" "dss" "cantian" "ct_om" "cantian_exporter" "mysql" "logicrep") +START_ORDER=("cms" "dss" "cantian" "ct_om" "cantian_exporter" "logicrep") +STOP_ORDER=("logicrep" "cms" "dss" "cantian" "ct_om" "cantian_exporter") +UNINSTALL_ORDER=("ct_om" "cantian" "dss" "cms" "dbstor" "logicrep") +BACKUP_ORDER=("cantian" "dss" "cms" "ct_om" "dbstor" ) +CHECK_STATUS=("cantian" "cms" "dss" "ct_om" "cantian_exporter") +PRE_UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "dss" "cantian" "mysql" "logicrep") +UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "dss" "cantian" "mysql" "logicrep") +POST_UPGRADE_ORDER=("ct_om" "cantian_exporter" "dbstor" "cms" "dss" "cantian" "mysql") ROLLBACK_ORDER=("dbstor" "cms" "cantian" "mysql" "ct_om" "cantian_exporter" "logicrep") INIT_CONTAINER_ORDER=("dbstor" "cms" "cantian" "logicrep") DIR_LIST=(/opt/cantian/cms /opt/cantian/cantian /opt/cantian/dbstor /opt/cantian/mysql ${CURRENT_FILE_PATH}/inspection/inspection_scripts/kernal ${CURRENT_FILE_PATH}/inspection/inspection_scripts/cms ${CURRENT_FILE_PATH}/inspection/inspection_scripts/ct_om) \ No newline at end of file diff --git a/pkg/deploy/action/install.sh b/pkg/deploy/action/install.sh index d0bffcb9856f5d98d0fed088f746731b689f323a..3815d7d6b0530300ca27f6ad361e1e39e0f0c7ff 100644 --- a/pkg/deploy/action/install.sh +++ b/pkg/deploy/action/install.sh @@ -36,6 +36,11 @@ NFS_TIMEO=50 source ${CURRENT_PATH}/log4sh.sh source ${FILE_MOD_FILE} +declare -A use_dorado +use_dorado=(["combined"]=1 ["dbstor"]=1) +use_file=(["file"]=1) +use_dss=(["dss"]=1) + # 适配欧拉系统，nologin用户没有执行ping命令的权限 chmod u+s /bin/ping @@ -91,7 +96,7 @@ function correct_files_mod() { # 获取用户输入用户名密码 function enter_pwd() { - if [[ x"${deploy_mode}" != x"file" ]];then + if [[ ${use_dorado["${deploy_mode}"]} ]];then read -p "please enter dbstor_user: " dbstor_user echo "dbstor_user is: ${dbstor_user}" @@ -142,11 +147,20 @@ function checkMountNFS() { # 配置ctmgruser sudo权限 function config_sudo() { - cantian_sudo="cantian ALL=(root) NOPASSWD:/usr/bin/chrt,/opt/cantian/action/docker/get_pod_ip_info.py" - cat /etc/sudoers | grep ^cantian + cantian_sudo="cantian ALL=(root) NOPASSWD:/usr/bin/chrt,/opt/cantian/action/docker/get_pod_ip_info.py,/usr/sbin/setcap" + cat /etc/sudoers | grep "cantian ALL" if [[ -n $? ]];then sed -i '/^cantian*/d' /etc/sudoers fi + echo "${cantian_sudo}" >> /etc/sudoers + + deploy_user_sudo="${deploy_user} ALL=(root) NOPASSWD:/usr/sbin/setcap" + cat /etc/sudoers | grep "${deploy_user_sudo} ALL" + if [[ -n $? ]];then + sed -i "/^${deploy_user}*/d" /etc/sudoers + fi + echo "${deploy_user_sudo}" >> /etc/sudoers + SERVICE_NAME=$(printenv SERVICE_NAME) cat /etc/sudoers | grep SERVICE_NAME if [[ $? -ne 0 ]];then @@ -155,7 +169,7 @@ function config_sudo() { echo "export SERVICE_NAME=${SERVICE_NAME}" >> /home/cantian/.bashrc fi fi - echo "${cantian_sudo}" >> /etc/sudoers + local chmod_script="/opt/cantian/action/change_log_priority.sh" ctmgruser_sudo="ctmgruser ALL=(root) NOPASSWD:${chmod_script}" cat /etc/sudoers | grep ctmgruser @@ -253,7 +267,7 @@ function check_port() { function rpm_check(){ local count=2 - if [ x"${deploy_mode}" != x"file" ];then + if [[ ${use_dorado["${deploy_mode}"]} ]];then count=3 fi rpm_pkg_count=$(ls "${CURRENT_PATH}"/../repo | wc -l) @@ -371,7 +385,7 @@ function install_rpm() rpm -ivh --replacepkgs ${RPM_PATH} --nodeps --force tar -zxf ${RPM_UNPACK_PATH_FILE}/Cantian-RUN-CENTOS-64bit.tar.gz -C ${RPM_PACK_ORG_PATH} - if [ x"${deploy_mode}" != x"file" ];then + if [[ ${use_dorado["${deploy_mode}"]} ]];then install_dbstor if [ $? -ne 0 ];then sh ${CURRENT_PATH}/uninstall.sh ${config_install_type} @@ -381,6 +395,10 @@ function install_rpm() chmod -R 750 ${RPM_PACK_ORG_PATH}/Cantian-RUN-CENTOS-64bit chown ${cantian_user}:${cantian_group} -hR ${RPM_PACK_ORG_PATH}/ chown root:root ${RPM_PACK_ORG_PATH} + if [[ ${deploy_mode} == "dss" ]];then + cp ${RPM_PACK_ORG_PATH}/Cantian-RUN-CENTOS-64bit/lib/* /usr/lib64/ + chown ${cantian_user}:${cantian_group} -hR /usr/lib64/libze* + fi } function show_cantian_version() { @@ -401,28 +419,23 @@ EOF # 检查dbstor的user与pwd是否正确 function check_dbstor_usr_passwd() { - if [[ x"${deploy_mode}" != x"file" ]];then - logAndEchoInfo "check username and password of dbstor. [Line:${LINENO}, File:${SCRIPT_NAME}]" - su -s /bin/bash - "${cantian_user}" -c "sh ${CURRENT_PATH}/dbstor/check_usr_pwd.sh" - install_result=$? - if [ ${install_result} -ne 0 ]; then - logAndEchoError "check dbstor passwd failed, possible reasons: - 1 username or password of dbstor storage service is incorrect. - 2 cgw create link failed. - 3 ip address of dbstor storage service is incorrect. - please contact the engineer to solve. [Line:${LINENO}, File:${SCRIPT_NAME}]" - uninstall - exit 1 - else - logAndEchoInfo "user and password of dbstor check pass. [Line:${LINENO}, File:${SCRIPT_NAME}]" - fi + logAndEchoInfo "check username and password of dbstor. [Line:${LINENO}, File:${SCRIPT_NAME}]" + su -s /bin/bash - "${cantian_user}" -c "sh ${CURRENT_PATH}/dbstor/check_usr_pwd.sh" + install_result=$? + if [ ${install_result} -ne 0 ]; then + logAndEchoError "check dbstor passwd failed, possible reasons: + 1 username or password of dbstor storage service is incorrect. + 2 cgw create link failed. + 3 ip address of dbstor storage service is incorrect. + please contact the engineer to solve. [Line:${LINENO}, File:${SCRIPT_NAME}]" + uninstall + exit 1 + else + logAndEchoInfo "user and password of dbstor check pass. [Line:${LINENO}, File:${SCRIPT_NAME}]" fi } function check_dbstore_client_compatibility() { - if [[ x"${deploy_mode}" == x"file" ]]; then - return 0 - fi logAndEchoInfo "begin to check dbstore client compatibility." if [ ! -f "${DBSTORE_CHECK_FILE}" ];then logAndEchoError "${DBSTORE_CHECK_FILE} file is not exists." @@ -451,7 +464,7 @@ function mount_fs() { fi mkdir -m 770 -p /mnt/dbdata/remote/metadata_${storage_metadata_fs}/node${node_id} chown ${deploy_user}:${cantian_common_group} /mnt/dbdata/remote/metadata_${storage_metadata_fs}/node${node_id} - if [[ "${cantian_in_container}" != "0" ]] || [[ x"${deploy_mode}" == x"dbstor" ]]; then + if [[ "${cantian_in_container}" != "0" ]] || [[ x"${deploy_mode}" == x"dbstor" ]] || [[ x"${deploy_mode}" == x"dss" ]]; then return 0 fi @@ -464,7 +477,7 @@ function mount_fs() { fi metadata_logic_ip=`python3 ${CURRENT_PATH}/get_config_info.py "metadata_logic_ip"` - if [[ x"${deploy_mode}" != x"file" ]]; then + if [[ ${use_dorado["${deploy_mode}"]} ]]; then kerberos_type=`python3 ${CURRENT_PATH}/get_config_info.py "kerberos_key"` mount -t nfs -o sec="${kerberos_type}",timeo=${NFS_TIMEO},nosuid,nodev ${metadata_logic_ip}:/${storage_metadata_fs} /mnt/dbdata/remote/metadata_${storage_metadata_fs} else @@ -480,7 +493,7 @@ function mount_fs() { check_port sysctl fs.nfs.nfs_callback_tcpport="${NFS_PORT}" > /dev/null 2>&1 if [[ ${storage_archive_fs} != '' ]]; then - if [[ x"${deploy_mode}" != x"file" ]]; then + if [[ ${use_dorado["${deploy_mode}"]} ]]; then mount -t nfs -o sec="${kerberos_type}",timeo=${NFS_TIMEO},nosuid,nodev ${archive_logic_ip}:/${storage_archive_fs} /mnt/dbdata/remote/archive_${storage_archive_fs} archive_result=$? else @@ -590,7 +603,7 @@ if [[ x"${is_single}" == x"cantiand_with_mysql_in_cluster" ]];then sed -i "s/cantian_group=\"cantian\"/cantian_group=\"${deploy_group}\"/g" "${CURRENT_PATH}"/env.sh fi -if [[ x"${deploy_mode}" == x"file" ]];then +if [[ ${use_dorado["${deploy_mode}"]} -ne 1 ]];then python3 "${CURRENT_PATH}"/modify_env.py if [ $? -ne 0 ];then echo "Current deploy mode is ${deploy_mode}, modify env.sh failed." @@ -741,6 +754,8 @@ if [[ ${config_install_type} = 'override' ]]; then storage_share_fs=`python3 ${CURRENT_PATH}/get_config_info.py "storage_share_fs"` storage_archive_fs=`python3 ${CURRENT_PATH}/get_config_info.py "storage_archive_fs"` storage_metadata_fs=`python3 ${CURRENT_PATH}/get_config_info.py "storage_metadata_fs"` + mkdir -m 750 -p /mnt/dbdata/remote/share_${storage_share_fs} + chown ${cantian_user}:${cantian_group} /mnt/dbdata/remote/share_${storage_share_fs} # 创建公共路径 mkdir -m 755 -p /opt/cantian/logs @@ -811,6 +826,10 @@ cp -rfp ${CURRENT_PATH}/dbstor /opt/cantian/action # 适配开源场景，使用file，不使用dbstore，提前安装参天rpm包 install_rpm +if [[ x"${deploy_mode}" == x"dss" ]];then + chown "${cantian_user}":"${cantian_group}" /dev/*-disk* +fi + # 调用各模块安装脚本，如果有模块安装失败直接退出，不继续安装接下来的模块 logAndEchoInfo "Begin to install. [Line:${LINENO}, File:${SCRIPT_NAME}]" for lib_name in "${INSTALL_ORDER[@]}" @@ -840,7 +859,7 @@ do exit 1 fi fi - if [[ "${cantian_in_container}" == "0" ]]; then + if [[ "${cantian_in_container}" == "0" ]] && [[ ${use_dorado["${deploy_mode}"]} ]]; then check_dbstor_usr_passwd # 检查dbstore client 与server端是否兼容 check_dbstore_client_compatibility diff --git a/pkg/deploy/action/pre_install.py b/pkg/deploy/action/pre_install.py index 76b8dd93a417240c22f8897a0f52aaffa4437b3e..147333fbf32dc7d1e06f62d14542fe461c1ec6d0 100644 --- a/pkg/deploy/action/pre_install.py +++ b/pkg/deploy/action/pre_install.py @@ -52,11 +52,13 @@ kernel_element = { 'VARIANT_MEMORY_AREA_SIZE', '_INDEX_BUFFER_SIZE' } +use_dbstor = ["dbstor", "combine"] UnitConversionInfo = collections.namedtuple('UnitConversionInfo', ['tmp_gb', 'tmp_mb', 'tmp_kb', 'key', 'value', 'sga_buff_size', 'temp_buffer_size', 'data_buffer_size', 'shared_pool_size', 'log_buffer_size']) + class ConfigChecker: """ 对参天安装的配置文件中内容进行校验的反射类 @@ -113,7 +115,7 @@ class ConfigChecker: @staticmethod def deploy_mode(value): - deploy_mode_enum = {"file", "combined", "dbstor"} + deploy_mode_enum = {"file", "combined", "dbstor", "dss"} if value not in deploy_mode_enum: return False @@ -327,6 +329,10 @@ class CheckInstallConfig(CheckBase): 'MAX_ARCH_FILES_SIZE', 'mysql_in_container', 'mysql_metadata_in_cantian', 'storage_logic_ip', 'deploy_mode', 'mes_ssl_switch', 'cantian_in_container', 'deploy_policy' } + self.dss_config_key = { + 'deploy_user', 'node_id', 'cms_ip', 'db_type', 'cantian_in_container', + 'MAX_ARCH_FILES_SIZE', 'mysql_in_container', 'mysql_metadata_in_cantian', + 'deploy_mode', 'mes_ssl_switch', "redo_num", "redo_size"} self.dbstore_config_key = { 'cluster_name', 'cantian_vlan_ip', 'storage_vlan_ip', 'link_type', 'storage_dbstore_page_fs', 'kerberos_key', 'cluster_id', 'mes_type', "vstore_id", "dbstore_fs_vstore_id" @@ -601,15 +607,14 @@ class CheckInstallConfig(CheckBase): if not ret: LOG.error("init deploy policy failed") return False + if install_config_params["deploy_mode"] == "dss": + self.config_key = self.dss_config_key self.install_config_params_init(install_config_params) self.cluster_name = install_config_params.get("cluster_name") - # 不开启归档时不检查归档连通性 - if install_config_params.get("storage_archive_fs") == "": - ping_check_element.remove("archive_logic_ip") - if install_config_params['deploy_mode'] != "file": + if install_config_params['deploy_mode'] in use_dbstor: self.config_key.remove("storage_logic_ip") self.config_key.update(self.dbstore_config_key) ping_check_element.remove("storage_logic_ip") @@ -627,15 +632,29 @@ class CheckInstallConfig(CheckBase): self.config_params['mes_type'] = "TCP" self.config_key.update(self.file_config_key) - if install_config_params['cantian_in_container'] != '0': + # 不开启归档时不检查归档连通性 + if install_config_params.get("storage_archive_fs") == "": + ping_check_element.remove("archive_logic_ip") + + if install_config_params.get("cantian_in_container", "0") != '0': ip_check_element.remove('cms_ip') ping_check_element.remove("cms_ip") ip_check_element.remove("cantian_vlan_ip") ping_check_element.remove("cantian_vlan_ip") - if install_config_params['archive_logic_ip'] == "" \ - and install_config_params['share_logic_ip'] == "" \ - and install_config_params['metadata_logic_ip'] == "": + if install_config_params["deploy_mode"] == "dss": + ip_check_element.remove("storage_vlan_ip") + ip_check_element.remove("cantian_vlan_ip") + ping_check_element.remove("cantian_vlan_ip") + ping_check_element.remove("storage_vlan_ip") + ping_check_element.remove("share_logic_ip") + ping_check_element.remove("metadata_logic_ip") + ping_check_element.remove("storage_logic_ip") + + if (install_config_params.get("archive_logic_ip", "") == "" + and install_config_params.get('share_logic_ip', "") == "" + and install_config_params.get('metadata_logic_ip', "") == "" + and install_config_params['deploy_mode'] in use_dbstor): install_config_params['archive_logic_ip'] = self.cluster_name install_config_params['share_logic_ip'] = self.cluster_name install_config_params['metadata_logic_ip'] = self.cluster_name @@ -646,7 +665,7 @@ class CheckInstallConfig(CheckBase): if not self.check_install_config_params(install_config_params): return False - if (install_config_params['deploy_mode'] != "file" and + if (install_config_params['deploy_mode'] in use_dbstor and not self.check_storage_cantian_vlan_ip_scale(install_config_params)): return False @@ -662,13 +681,13 @@ class CheckInstallConfig(CheckBase): except Exception as error: LOG.error('write config param to config_param.json failed, error: %s', str(error)) return False - if install_config_params['cantian_in_container'] == '0': + if install_config_params.get("'cantian_in_container'", "0") == '0': try: self.write_result_to_json() except Exception as error: LOG.error('write config param to deploy_param.json failed, error: %s', str(error)) return False - if install_config_params['cantian_in_container'] != '0': + if install_config_params.get('cantian_in_container', "0") != '0': if not self.check_cantian_mem_spec(): return False @@ -691,12 +710,15 @@ class CheckInstallConfig(CheckBase): install_config_params['deploy_mode'] = "combined" if 'dbstore_fs_vstore_id' not in install_config_params.keys(): install_config_params['dbstore_fs_vstore_id'] = "0" - if install_config_params.get("mes_ssl_switch") == True and install_config_params.get("cantian_in_container", "-1") == "0": + if (install_config_params.get("mes_ssl_switch") and + install_config_params.get("cantian_in_container", "-1") == "0"): self.config_key.update(self.mes_type_key) if 'db_type' not in install_config_params.keys(): install_config_params['db_type'] = '0' if 'mysql_metadata_in_cantian' not in install_config_params.keys(): install_config_params['mysql_metadata_in_cantian'] = True + if 'cantian_in_container' not in install_config_params.keys(): + install_config_params['cantian_in_container'] = "0" def parse_policy_config_file(self): policy_path = os.path.join(dir_name, "deploy_policy_config.json") diff --git a/pkg/deploy/action/start.sh b/pkg/deploy/action/start.sh index d65379a96ab273a37ffa696636d3fedaa76ec105..b64ee3b3daa47e07694c09adf140044b71fd9edb 100644 --- a/pkg/deploy/action/start.sh +++ b/pkg/deploy/action/start.sh @@ -115,7 +115,7 @@ if [ "$cantian_in_container" -eq 0 ]; then done fi -chmod 660 /dev/shm/cantian* +chmod 660 /dev/shm/cantian* > /dev/null 2>&1 chown -hR "${cantian_user}":"${deploy_group}" /dev/shm/cantian* logAndEchoInfo "start success" exit 0 \ No newline at end of file diff --git a/pkg/deploy/action/uninstall.sh b/pkg/deploy/action/uninstall.sh index 06e0f97a1bae6ebac6df440377d124ad28a3dcf4..8fa58a84693fb530b09b0b87bf862994187bc126 100644 --- a/pkg/deploy/action/uninstall.sh +++ b/pkg/deploy/action/uninstall.sh @@ -323,6 +323,7 @@ if [[ ${uninstall_type} = 'override' ]]; then rm -rf /opt/cantian/image /opt/cantian/action /opt/cantian/config rm -rf /usr/local/bin/show fi - +# 清理共享内存 +ipcrm -a logAndEchoInfo "uninstall finished" exit 0 diff --git a/pkg/install/script/cluster/cluster.sh b/pkg/install/script/cluster/cluster.sh index 58c0c03f0c2b3d5f7165ee1b472b31cdb0f3a6ce..ae4932efc7564f6186dd1432238e7102d4b8ad40 100644 --- a/pkg/install/script/cluster/cluster.sh +++ b/pkg/install/script/cluster/cluster.sh @@ -52,6 +52,7 @@ function check_process() return 0 else echo "res_count= ${res_count}" + echo "RES_EAGAIN" return 1 fi fi @@ -64,7 +65,8 @@ function start_cantian() { numactl --hardware > /dev/null 2>&1 if [ $? -eq 0 ]; then OS_ARCH=$(uname -i) - if [[ ${OS_ARCH} =~ "aarch64" ]]; then + deploy_mode=$(python3 /opt/cantian/action/cantian/get_config_info.py "deploy_mode") + if [[ ${OS_ARCH} =~ "aarch64" ]] && [[ x"${deploy_mode}" != x"dss" ]]; then CPU_CORES_NUM=`cat /proc/cpuinfo |grep "architecture" |wc -l` CPU_CORES_NUM=$((CPU_CORES_NUM - 1)) numactl_str="numactl -C 0-1,6-11,16-"${CPU_CORES_NUM}" " @@ -132,8 +134,8 @@ function stop_cantian() { echo "RES_SUCCESS" exit 0 else - echo "RES_MULTI" - exit 1 + echo "RES_EAGAIN" + exit 3 fi fi } @@ -159,8 +161,8 @@ function stop_cantian_by_force() { echo "RES_SUCCESS" exit 0 else - echo "RES_FAILED" - exit 1 + echo "RES_EAGAIN" + exit 3 fi fi } diff --git a/pkg/src/cluster/dtc_buffer.c b/pkg/src/cluster/dtc_buffer.c index a913cddc6ff760ab0cbb0c1bfd7a33b4ca310e63..d1c51079d45967d85ba449e308511c1574d75886 100644 --- a/pkg/src/cluster/dtc_buffer.c +++ b/pkg/src/cluster/dtc_buffer.c @@ -176,7 +176,7 @@ static status_t dtc_buf_finish(knl_session_t *session, buf_read_assist_t *ra, bu buf_log_enter_page(session, ctrl, ra->mode, ra->options); if (DTC_BUF_PREFETCH_EXTENT(ra->read_num) && session->kernel->attr.enable_asynch) { - knl_panic(0); + return CT_SUCCESS; if (buf_try_prefetch_next_ext(session, ctrl) != CT_SUCCESS) { CT_LOG_RUN_WAR("failed to prefetch next extent file : %u , page: %llu", (uint32)ctrl->page_id.file, (uint64)ctrl->page_id.page); diff --git a/pkg/src/cluster/dtc_dc.c b/pkg/src/cluster/dtc_dc.c index 9b551bc3f3a7a0872a2343435782df5bfbe60827..afd876924fb190a3d68428fe139d63a90859f06a 100644 --- a/pkg/src/cluster/dtc_dc.c +++ b/pkg/src/cluster/dtc_dc.c @@ -571,13 +571,13 @@ status_t dtc_process_heap_extend(knl_session_t *session, char *data, uint8 src_i return CT_ERROR; } if (bcast->extending == CT_TRUE && heap->extending) { - CT_LOG_DEBUG_ERR( + CT_LOG_DEBUG_INF( "[DTC][process_heap_extend] current node is doing heap extend, uid/table_id/part/subpart:[%d-%d-%u-%u], " "extending:%d, compacting:%d, extend owner:%d, wait ticks:%u", bcast->uid, bcast->table_id, bcast->part_loc.part_no, bcast->part_loc.subpart_no, bcast->extending, bcast->compacting, heap->extend_owner, heap->wait_ticks); dc_close(&dc); - return CT_ERROR; + return CT_SUCCESS; } heap->extending = bcast->extending; heap->compacting = bcast->compacting; diff --git a/pkg/src/cluster/dtc_reform.c b/pkg/src/cluster/dtc_reform.c index 1090fdc27429f83659738940836fad9ea0b6fb5e..3d624b6fa3186fe1b72793046117708519351ea2 100644 --- a/pkg/src/cluster/dtc_reform.c +++ b/pkg/src/cluster/dtc_reform.c @@ -845,6 +845,7 @@ status_t rc_archive_log_offline_node(arch_proc_context_t *proc_ctx, uint32 node_ bool32 rc_need_archive_log(void) { + return CT_FALSE; knl_session_t *session = (knl_session_t *)g_rc_ctx->session; if (session->kernel->db.ctrl.core.log_mode != ARCHIVE_LOG_ON || !DB_IS_PRIMARY(&session->kernel->db)) { return CT_FALSE; diff --git a/pkg/src/cluster/dtc_tran.c b/pkg/src/cluster/dtc_tran.c index e6119a07be8272f05bf3a8e5b279f715373a2e90..a84529e5004f863b66b0b85de9e08c3152964f84 100644 --- a/pkg/src/cluster/dtc_tran.c +++ b/pkg/src/cluster/dtc_tran.c @@ -92,7 +92,7 @@ void dtc_flush_log(knl_session_t *session, page_id_t page_id) buf_bucket_t *bucket = buf_find_bucket(session, page_id); cm_spin_lock(&bucket->lock, &session->stat->spin_stat.stat_bucket); buf_ctrl_t *ctrl = buf_find_from_bucket(bucket, page_id); - bool32 need_flush = (ctrl->is_dirty || ctrl->is_marked) && DAAC_NEED_FLUSH_LOG(session, ctrl); + bool32 need_flush = (ctrl != NULL) && (ctrl->is_dirty || ctrl->is_marked) && DAAC_NEED_FLUSH_LOG(session, ctrl); cm_spin_unlock(&bucket->lock); if (need_flush) { diff --git a/pkg/src/cms/CMakeLists.txt b/pkg/src/cms/CMakeLists.txt index e28d80dd0c347b5f66a1010ff7335b9e63a3a248..ea2d0f50810ef55531f03ec08e703de3eb0d64c9 100644 --- a/pkg/src/cms/CMakeLists.txt +++ b/pkg/src/cms/CMakeLists.txt @@ -4,17 +4,20 @@ include_directories( "${CANTIANDB_LIBRARY}/xnet/include" "${CANTIANDB_SRC}/upgrade_check" "${CANTIANDB_SRC}/cms/cms" + "${CANTIANDB_SRC}/cms/cbb" ) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/interface INTERFACE_SRC) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/cms CMS_SRC) +aux_source_directory(${CANTIANDB_SRC}/cms/cbb CBB_SRC) - -set(CMS_TOTAL_SRC ${INTERFACE_SRC} ${CMS_SRC}) +set(CMS_TOTAL_SRC ${INTERFACE_SRC} ${CMS_SRC} ${CBB_SRC}) # generate static library add_library(zecms STATIC ${INTERFACE_SRC}) add_library(zecmssrc STATIC ${CMS_SRC}) +add_library(dsslock SHARED ${CBB_SRC}) + add_executable(cms ${CMS_TOTAL_SRC} ${VERSION_SRC}) if (USE_FUZZASAN) add_compile_options(-fsanitize-coverage=trace-pc) @@ -22,5 +25,7 @@ if (USE_FUZZASAN) else() target_link_libraries(cms PRIVATE zemes zecms zeprotocol zecommon m rt pthread dl z) endif(USE_FUZZASAN) +target_link_libraries(dsslock PRIVATE zecommon m rt pthread dl z) + target_compile_definitions(cms PRIVATE -DCT_LIB_VERSION=${DD_CT_LIB_VERSION}) target_compile_definitions(cms PRIVATE -DGETDBVERSION=${DD_GETDBVERSION}) \ No newline at end of file diff --git a/pkg/src/cms/cbb/cms_cbb.c b/pkg/src/cms/cbb/cms_cbb.c new file mode 100644 index 0000000000000000000000000000000000000000..7d082466f9f2cf627b3a49f26c8a823314013e15 --- /dev/null +++ b/pkg/src/cms/cbb/cms_cbb.c @@ -0,0 +1,140 @@ +#include "cm_defs.h" +#include "test_log.h" +#include "cms_cbb.h" +#include "cms_disklock.h" + +#define CM_MAX_RES_NAME_LENGTH 32 +#define CM_MAX_INST_COUNTS 64 +#define MAX_EXIT_STATUS 128 + +#define DSS_RES_DATA_LOCK_POS (1073741824) + +unsigned int g_lock_id = 0; + +int CmInit(unsigned int instance_id, const char *res_name, cm_notify_func_t func) +{ + LOG("CmInit ==> instance_id: %d, res_name: %s", instance_id, res_name); + int ret = cm_dl_alloc("/dev/gcc-disk", DSS_RES_DATA_LOCK_POS, instance_id); + if (ret == CM_INVALID_LOCK_ID) { + LOG("cm_dl_alloc failed with ret: %d", ret); + return CT_ERROR; + } + g_lock_id = ret; + LOG("CmInit completed, g_lock_id: %u", g_lock_id); + return CT_SUCCESS; +} + +char* CmGetResStats(void) +{ + LOG("CmGetResStats ==>"); + + const char *json_data = + "{" + "\"res_name\": \"example\"," + "\"version\": 0," + "\"inst_count\": 2," + "\"inst_status\": [" + "{" + "\"node_id\": 0," + "\"res_instance_id\": 0," + "\"is_work_member\": 1," + "\"status\": 1" + "}," + "{" + "\"node_id\": 1," + "\"res_instance_id\": 1," + "\"is_work_member\": 1," + "\"status\": 1" + "}" + "]" + "}"; + + char *result = (char *)malloc(strlen(json_data) + 1); + if (result == NULL) { + LOG("Memory allocation failed in CmGetResStats"); + return NULL; + } + + strcpy(result, json_data); + LOG("CmGetResStats completed, JSON data allocated."); + return result; +} + +void CmFreeResStats(char *res_stat) +{ + LOG("CmFreeResStats ==>"); + if (res_stat != NULL) { + res_stat = NULL; + LOG("res_stat memory freed."); + } +} + +int CmResLock(const char *lock_name) +{ + LOG("CmResLock ==> lock_name: %s, g_lock_id: %d", lock_name, g_lock_id); + int ret = cm_dl_lock(g_lock_id, 1000); + if (ret == 0) { + LOG("cm_dl_lock succeeded"); + return CT_SUCCESS; + } + + unsigned long long lockTime = 0; + ret = cm_dl_getlocktime(g_lock_id, &lockTime); + if (ret != 0) { + LOG("cm_dl_getlocktime failed with ret: %d", ret); + return CT_ERROR; + } + + int result = (int)(lockTime % MAX_EXIT_STATUS); + LOG("CmResLock completed, lockTime: %llu, result: %d", lockTime, result); + + return result == CT_SUCCESS ? CT_ERROR : result; +} + +int CmResUnlock(const char *lock_name) +{ + LOG("CmResUnlock ==> lock_name: %s, g_lock_id: %u", lock_name, g_lock_id); + int ret = cm_dl_unlock(g_lock_id); + + if (ret != 0) { + LOG("CmResUnlock failed with ret: %d", ret); + return CT_ERROR; + } + + LOG("CmResUnlock succeeded"); + return CT_SUCCESS; +} + +int CmResGetLockOwner(const char *lock_name, unsigned int *inst_id) +{ + LOG("CmResGetLockOwner ==> lock_name: %s, g_lock_id: %d", lock_name, g_lock_id); + if (inst_id == NULL) { + LOG("CmResGetLockOwner failed due to NULL inst_id pointer"); + return CT_ERROR; + } + + unsigned long long temp_inst_id; + int ret = cm_dl_getowner(g_lock_id, &temp_inst_id); + if (ret != 0) { + LOG("cm_dl_getowner failed with ret: %d", ret); + return ret; + } + + *inst_id = (unsigned int)temp_inst_id; + LOG("CmResGetLockOwner completed, inst_id: %u", *inst_id); + + return CT_SUCCESS; +} + +int CmResTransLock(const char *lock_name, unsigned int inst_id) +{ + LOG("CmResTransLock ==> lock_name: %s, g_lock_id: %d", lock_name, g_lock_id); + int ret = cm_dl_unlock(g_lock_id); + if (ret != 0) { + LOG("CmResTransLock failed to unlock, ret: %d", ret); + return CT_ERROR; + } + + LOG("CmResTransLock completed successfully"); + return CT_SUCCESS; +} \ No newline at end of file diff --git a/pkg/src/cms/cbb/cms_cbb.h b/pkg/src/cms/cbb/cms_cbb.h new file mode 100644 index 0000000000000000000000000000000000000000..5730615b388ba7ddc86a7cdedfa91dbb6742dfdf --- /dev/null +++ b/pkg/src/cms/cbb/cms_cbb.h @@ -0,0 +1,14 @@ +#ifndef CMS_CBB_H +#define CMS_CBB_H + +typedef void (*cm_notify_func_t)(void); + +int CmInit(unsigned int instance_id, const char *res_name, cm_notify_func_t func); +char* CmGetResStats(void); +void CmFreeResStats(char *res_stat); +int CmResLock(const char *lock_name); +int CmResUnlock(const char *lock_name); +int CmResGetLockOwner(const char *lock_name, unsigned int *inst_id); +int CmResTransLock(const char *lock_name, unsigned int inst_id); + +#endif \ No newline at end of file diff --git a/pkg/src/cms/cbb/cms_disklock.c b/pkg/src/cms/cbb/cms_disklock.c new file mode 100644 index 0000000000000000000000000000000000000000..fa5a4d08134010b4f32f9e7ab7bd2dfeae311bd4 --- /dev/null +++ b/pkg/src/cms/cbb/cms_disklock.c @@ -0,0 +1,500 @@ +#include +#include +#include +#include +#include +#include +#include "securec.h" +#include "cm_defs.h" +#include "test_log.h" +#include "cms_disklock.h" + +#define CM_BLOCK_SIZE (512) +#define CM_ALIGN_SIZE (8192) +#define CM_LOCK_FULL_SIZE (CM_BLOCK_SIZE * (CM_MAX_INST_COUNT + 1)) +#define CM_DL_MAGIC (0xFEDCBA9801234567ULL) +#define CM_DL_PROC_VER (1) +#define CM_MAX_RETRY_WAIT_TIME_MS (200) +#define NANOSECS_PER_SECOND_LL (1000) +#define NANOSECS_PER_MILLISECS_LL (1000000) + +typedef enum e_lockstatus { LS_NO_LOCK = 0, LS_PRE_LOCK = 1, LS_LOCKED = 2 } lockstatus_t; +typedef enum e_locktype { LT_NORMAL = 0, LT_LEASE = 1} locktype_t; +typedef enum e_checkperiod {CP_PRE_CHECK = 0, CP_CONFIRM = 1} checkperiod_t; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union u_dl_stat { + struct { + unsigned long long magic; + unsigned long long proc_ver; + unsigned long long inst_id; + unsigned long long locked; + unsigned long long lock_time; + unsigned long long unlock_time; + }; + struct { + char placeholder[CM_BLOCK_SIZE]; + }; +} dl_stat_t; + +typedef struct st_dl_stat { + unsigned long long peer_lock_time; + unsigned long long lock_hb_time; +} dl_hb_t; + +typedef struct st_dl_lock { + char path[CM_MAX_PATH_SIZE]; + unsigned long long offset; + unsigned long long inst_id; + int fd; + unsigned int lease_sec; + locktype_t type; + dl_stat_t *lock_stat; + dl_hb_t *hb; +} cm_dl_t; + +typedef struct st_dl_ctx { + cm_dl_t lock_info[CM_MAX_DISKLOCK_COUNT]; + pthread_mutex_t lock; +} dl_ctx_t; + +static dl_ctx_t g_dl_ctx; + +static int cm_dl_unlock_inner(unsigned int lock_id, unsigned long long inst_id); + +static inline unsigned long long cm_dl_now_ns() +{ + struct timespec tv; + (void)clock_gettime(CLOCK_MONOTONIC, &tv); + return (unsigned long long)tv.tv_sec * NANOSECS_PER_SECOND_LL + (unsigned long long)tv.tv_nsec; +} + +unsigned int cm_dl_alloc(const char *path, unsigned long long offset, unsigned long long inst_id) +{ + if (path == NULL) { + LOG("DL:invalid path[NULL]."); + return CM_INVALID_LOCK_ID; + } + + size_t len = strlen(path); + if (len == 0 || len > CM_MAX_PATH_SIZE - 1) { + LOG("DL:invalid path length."); + return CM_INVALID_LOCK_ID; + } + + if ((offset & (CM_ALIGN_SIZE - 1)) != 0) { + LOG("DL:invalid offset:not %d aligned.", CM_ALIGN_SIZE); + return CM_INVALID_LOCK_ID; + } + + if (inst_id >= CM_MAX_INST_COUNT) { + LOG("DL:invalid inst_id[%lld].", inst_id); + return CM_INVALID_LOCK_ID; + } + + if (pthread_mutex_lock(&g_dl_ctx.lock) != 0) { + LOG("DL:pthread_mutex_lock failed."); + return CM_INVALID_LOCK_ID; + } + + unsigned int id = 0; + for (; id < CM_MAX_DISKLOCK_COUNT; id++) { + if (g_dl_ctx.lock_info[id].fd <= 0) { + g_dl_ctx.lock_info[id].fd = CT_MAX_INT32; + break; + } + } + + if (pthread_mutex_unlock(&g_dl_ctx.lock) != 0) { + LOG("DL:pthread_mutex_unlock failed."); + return CM_INVALID_LOCK_ID; + } + + if (id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:insufficient lock area."); + return CM_INVALID_LOCK_ID; + } + + int fd = open(path, O_RDWR | O_DIRECT | O_SYNC); + if (fd <= 0) { + (void)close(fd); + g_dl_ctx.lock_info[id].fd = 0; + LOG("DL:open path failed:%d,%s.", errno, strerror(errno)); + return CM_INVALID_LOCK_ID; + } + + int64 size = lseek64(fd, 0, SEEK_END); + if (size < (off_t)offset + CM_LOCK_FULL_SIZE) { + (void)close(fd); + g_dl_ctx.lock_info[id].fd = 0; + LOG("DL:insufficient path size:%lld,%s.", size, strerror(errno)); + return CM_INVALID_LOCK_ID; + } + + dl_stat_t *lock_stat = (dl_stat_t *)aligned_alloc(CM_BLOCK_SIZE, CM_BLOCK_SIZE * (CM_MAX_INST_COUNT + 1)); + if (lock_stat == NULL) { + (void)close(fd); + g_dl_ctx.lock_info[id].fd = 0; + LOG("DL:insufficient memory."); + return CM_INVALID_LOCK_ID; + } + + errno_t errcode = strcpy_sp(g_dl_ctx.lock_info[id].path, CM_MAX_PATH_SIZE, path); + if (errcode != EOK) { + (void)close(fd); + g_dl_ctx.lock_info[id].fd = 0; + free(lock_stat); + LOG("DL:strcpy_sp failed."); + return CM_INVALID_LOCK_ID; + } + + g_dl_ctx.lock_info[id].lock_stat = lock_stat; + g_dl_ctx.lock_info[id].hb = NULL; + g_dl_ctx.lock_info[id].fd = fd; + g_dl_ctx.lock_info[id].offset = offset; + g_dl_ctx.lock_info[id].inst_id = inst_id; + g_dl_ctx.lock_info[id].type = LT_NORMAL; + + LOG("DL:cm_dl_alloc succeed:%s:%lld.", path, offset); + + return id; +} + +int cm_dl_dealloc(unsigned int lock_id) +{ + if (lock_id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:invalid lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + if (lock_info->fd <= 0) { + LOG("DL:invalid lock not ready,lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + LOG("DL:cm_dl_dealloc:%s:%lld.", lock_info->path, lock_info->offset); + + if (lock_info->lock_stat != NULL) { + free(lock_info->lock_stat); + lock_info->lock_stat = NULL; + } + + if(lock_info->hb != NULL) { + free(lock_info->hb); + lock_info->hb = NULL; + } + + (void)close(lock_info->fd); + lock_info->fd = 0; + + return CT_SUCCESS; +} + +static int cm_dl_check_lock(unsigned int lock_id, checkperiod_t checkperiod) +{ + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + dl_stat_t *lock_stat = lock_info->lock_stat; + + ssize_t size = pread(lock_info->fd, lock_stat, CM_LOCK_FULL_SIZE, (off_t)lock_info->offset); + if (size != CM_LOCK_FULL_SIZE) { + LOG("DL:read path failed:%d,%s.", errno, strerror(errno)); + return CM_DL_ERR_IO; + } + + for (unsigned long long inst_id = 0; inst_id < CM_MAX_INST_COUNT; inst_id++) { + if (inst_id == lock_info->inst_id) { + continue; + } + + lock_stat = &lock_info->lock_stat[inst_id + 1]; + if (lock_stat->magic != CM_DL_MAGIC) { + continue; + } + + if (lock_stat->locked == LS_NO_LOCK) { + continue; + } + + if (lock_info->type == LT_NORMAL) { + return CM_DL_ERR_OCCUPIED; + } else if (lock_info->type == LT_LEASE) { + LOG("DL:check lease:%d.", checkperiod); + if (checkperiod == CP_CONFIRM) { + LOG("DL:return CM_DL_ERR_OCCUPIED lease:%d.", checkperiod); + return CM_DL_ERR_OCCUPIED; + } + + dl_hb_t *hb = &lock_info->hb[inst_id]; + LOG("DL:lock_time=%lld,peer_lock_time=%lld.", lock_stat->lock_time, hb->peer_lock_time); + if (lock_stat->lock_time != hb->peer_lock_time) { + hb->peer_lock_time = lock_stat->lock_time; + hb->lock_hb_time = cm_dl_now_ns(); + LOG("DL:update hb:peer_lock_time=%lld,lock_hb_time=%lld", hb->peer_lock_time, hb->lock_hb_time); + } + + LOG("DL:now=%lld,lock_hb_time=%lld,lease_ns=%lld.", + cm_dl_now_ns(), + hb->lock_hb_time, + lock_info->lease_sec * NANOSECS_PER_SECOND_LL); + + if (cm_dl_now_ns() - hb->lock_hb_time > lock_info->lease_sec * NANOSECS_PER_SECOND_LL) { + LOG("DL:release lock,inst_id=%llu", inst_id); + cm_dl_unlock_inner(lock_id, inst_id); + } else { + LOG("DL:CM_DL_ERR_OCCUPIED"); + return CM_DL_ERR_OCCUPIED; + } + } + } + + return CT_SUCCESS; +} + +static int cm_dl_lock_inner(unsigned int lock_id) +{ + int ret = 0; + + ret = cm_dl_check_lock(lock_id, CP_PRE_CHECK); + if (ret != CT_SUCCESS) { + return ret; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + dl_stat_t *lock_stat = &lock_info->lock_stat[lock_info->inst_id + 1]; + + lock_stat->magic = CM_DL_MAGIC; + lock_stat->proc_ver = CM_DL_PROC_VER; + lock_stat->inst_id = lock_info->inst_id; + lock_stat->lock_time = cm_dl_now_ns(); + lock_stat->locked = LS_PRE_LOCK; + + ssize_t size = pwrite( + lock_info->fd, lock_stat, CM_BLOCK_SIZE, (off_t)(lock_info->offset + CM_BLOCK_SIZE * (lock_info->inst_id + 1))); + if (size != CM_BLOCK_SIZE) { + LOG("DL:write path failed:size=%lu,%d,%s.", size, errno, strerror(errno)); + return CM_DL_ERR_IO; + } + + ret = cm_dl_check_lock(lock_id, CP_CONFIRM); + if (ret != CT_SUCCESS) { + (void)cm_dl_unlock_inner(lock_id, lock_info->inst_id); + return ret; + } + + lock_stat->locked = LS_LOCKED; + size = pwrite( + lock_info->fd, lock_stat, CM_BLOCK_SIZE, (off_t)(lock_info->offset + CM_BLOCK_SIZE * (lock_info->inst_id + 1))); + if (size != CM_BLOCK_SIZE) { + (void)cm_dl_unlock_inner(lock_id, lock_info->inst_id); + LOG("DL:write path failed:size=%lu,%d,%s.", size, errno, strerror(errno)); + return CM_DL_ERR_IO; + } + + return CT_SUCCESS; +} + +int cm_dl_lock(unsigned int lock_id, int timeout_ms) +{ + int ret; + + if (lock_id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:invalid lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + if (lock_info->fd <= 0) { + LOG("DL:invalid lock not ready,lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + unsigned long long start = cm_dl_now_ns(); + do { + ret = cm_dl_lock_inner(lock_id); + if (ret != CM_DL_ERR_OCCUPIED) { + break; + } + + unsigned long long now = cm_dl_now_ns(); + if (timeout_ms >= 0) { + if (now - start > (unsigned long long)timeout_ms * NANOSECS_PER_MILLISECS_LL) { + return CM_DL_ERR_TIMEOUT; + } + } + + unsigned long long random_time = + ((start + now) & (CM_MAX_INST_COUNT - 1)) * (CM_MAX_RETRY_WAIT_TIME_MS / CM_MAX_INST_COUNT) + + lock_info->inst_id; + cm_sleep(random_time); + LOG("DL:wait for retry:%lldms.", random_time); + } while (CT_TRUE); + + return ret; +} + +static int cm_dl_unlock_inner(unsigned int lock_id, unsigned long long inst_id) +{ + if (inst_id >= CM_MAX_INST_COUNT) { + LOG("DL:invalid inst_id[%lld].", inst_id); + return CM_DL_ERR_INVALID_PARAM; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + dl_stat_t *lock_stat = &lock_info->lock_stat[inst_id + 1]; + + lock_stat->magic = CM_DL_MAGIC; + lock_stat->proc_ver = CM_DL_PROC_VER; + lock_stat->inst_id = inst_id; + lock_stat->unlock_time = cm_dl_now_ns(); + lock_stat->locked = LS_NO_LOCK; + + ssize_t size = + pwrite(lock_info->fd, lock_stat, CM_BLOCK_SIZE, (off_t)(lock_info->offset + CM_BLOCK_SIZE * (inst_id + 1))); + if (size != CM_BLOCK_SIZE) { + LOG("DL:write path failed:%d,%s.", errno, strerror(errno)); + return CM_DL_ERR_IO; + } + + return CT_SUCCESS; +} + +int cm_dl_unlock(unsigned int lock_id) +{ + if (lock_id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:invalid lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + if (lock_info->fd <= 0) { + LOG("DL:invalid lock not ready,lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + return cm_dl_unlock_inner(lock_id, lock_info->inst_id); +} + +int cm_dl_clean(unsigned int lock_id, unsigned long long inst_id) +{ + if (lock_id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:invalid lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + if (lock_info->fd <= 0) { + LOG("DL:invalid lock not ready,lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + return cm_dl_unlock_inner(lock_id, inst_id); +} + +static int cm_dl_getlockstat(unsigned int lock_id, dl_stat_t **lock_stat) +{ + if (lock_id >= CM_MAX_DISKLOCK_COUNT) { + LOG("DL:invalid lock_id:%u.", lock_id); + return CM_DL_ERR_INVALID_LOCK_ID; + } + + if (lock_stat == NULL) { + LOG("DL:invalid lock_stat."); + return CM_DL_ERR_INVALID_PARAM; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + dl_stat_t *lock_stat_x = lock_info->lock_stat; + + ssize_t size = pread(lock_info->fd, lock_stat_x, CM_LOCK_FULL_SIZE, (off_t)lock_info->offset); + if(size != CM_LOCK_FULL_SIZE) { + LOG("DL:read path failed:%d,%s.", errno, strerror(errno)); + return CM_DL_ERR_IO; + } + + *lock_stat = NULL; + for (unsigned long long x_inst_id = 0; x_inst_id < CM_MAX_INST_COUNT; x_inst_id++) { + if (lock_stat_x[x_inst_id + 1].locked == LS_LOCKED) { + if (*lock_stat == NULL) { + *lock_stat = &lock_stat_x[x_inst_id + 1]; + } else { + LOG("DL:This lock hash more than one owner:inst1=%lld,inst2=%lld.", (*lock_stat)->inst_id, x_inst_id); + return CM_DL_ERR_INVALID_LOCKSTAT; + } + } + } + + return CT_SUCCESS; +} + +int cm_dl_getowner(unsigned int lock_id, unsigned long long *inst_id) +{ + dl_stat_t *lock_stat = NULL; + int ret = cm_dl_getlockstat(lock_id, &lock_stat); + if (ret != CT_SUCCESS) { + return ret; + } + + if (lock_stat == NULL) { + *inst_id = CM_INVALID_INST_ID; + } else { + *inst_id = lock_stat->inst_id; + } + + return CT_SUCCESS; +} + +int cm_dl_getlocktime(unsigned int lock_id, unsigned long long *locktime) +{ + dl_stat_t *lock_stat = NULL; + int ret = cm_dl_getlockstat(lock_id, &lock_stat); + if (ret != CT_SUCCESS) { + return ret; + } + + if (lock_stat == NULL) { + *locktime = 0; + } else { + *locktime = lock_stat->lock_time; + } + + return CT_SUCCESS; +} + +unsigned int cm_dl_alloc_lease( + const char *path, unsigned long long offset, unsigned long long inst_id, unsigned int lease_sec) +{ + unsigned int lock_id = cm_dl_alloc(path, offset, inst_id); + if (lock_id == CM_INVALID_LOCK_ID) { + return CM_INVALID_LOCK_ID; + } + + dl_hb_t *hb = (dl_hb_t *)malloc(sizeof(dl_hb_t) * CM_MAX_INST_COUNT); + if (hb == NULL) { + cm_dl_dealloc(lock_id); + LOG("DL:insufficient memory."); + return CM_INVALID_LOCK_ID; + } + + errno_t errcode = memset_sp(hb, sizeof(dl_hb_t) * CM_MAX_INST_COUNT, 0, sizeof(dl_hb_t) * CM_MAX_INST_COUNT); + if (errcode != EOK) { + cm_dl_dealloc(lock_id); + free(hb); + LOG("DL:memset_sp failed."); + return CM_INVALID_LOCK_ID; + } + + cm_dl_t *lock_info = &g_dl_ctx.lock_info[lock_id]; + lock_info->type = LT_LEASE; + lock_info->lease_sec = lease_sec; + lock_info->hb = hb; + + return lock_id; +} + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/pkg/src/cms/cbb/cms_disklock.h b/pkg/src/cms/cbb/cms_disklock.h new file mode 100644 index 0000000000000000000000000000000000000000..12633cc5a133582b2c6d30a9082c9a9b98f25f78 --- /dev/null +++ b/pkg/src/cms/cbb/cms_disklock.h @@ -0,0 +1,101 @@ +#ifndef __CM_DISKLOCK_H__ +#define __CM_DISKLOCK_H__ + +#define CM_MAX_DISKLOCK_COUNT (16) +#define CM_MAX_PATH_SIZE (256) +#define CM_MAX_INST_COUNT (15) + +#define CM_DL_ERR_IO (1) +#define CM_DL_ERR_OCCUPIED (2) +#define CM_DL_ERR_INVALID_LOCK_ID (3) +#define CM_DL_ERR_TIMEOUT (4) +#define CM_DL_ERR_INVALID_PARAM (5) +#define CM_DL_ERR_INVALID_LOCKSTAT (6) + +#define CM_INVALID_LOCK_ID ((unsigned int)-1) +#define CM_INVALID_INST_ID ((unsigned long long)-1) + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Allocate a disk lock: reserve CM_MAX_INST_COUNT*(BLOCK_SIZE + 1) for the disk lock area. + * @param [in] path: disk path + * @param [in] offset: start position of the lock located, 8K aligned. + * @param [in] inst_id: base on 0, less than CM_MAX_INST_COUNT. + * @return >= 0 lock_id + * @return CM_INVALID_LOCK_ID fail + */ +unsigned int cm_dl_alloc(const char *path, unsigned long long offset, unsigned long long inst_id); + +/** + * Allocate a lease disk lock: reserve CM_MAX_INST_COUNT*(BLOCK_SIZE + 1) for the disk lock area. + * @param [in] path: disk path + * @param [in] offset: start position of the lock located, 8K aligned. + * @param [in] inst_id: base on 0, less than CM_MAX_INST_COUNT. + * @param [in] lease_sec: lease duration, unit second + * @return >= 0 lock_id + * @return CM_INVALID_LOCK_ID fail + */ +unsigned int cm_dl_alloc_lease( + const char *path, unsigned long long offset, unsigned long long inst_id, unsigned int lease_sec); + +/** + * Deallocate a disk lock. + * @param [in] lock_id: lock_id + * @return 0 success + * @return != 0 fail + */ +int cm_dl_dealloc(unsigned int lock_id); + +/** + * Try lock. + * @param [in] lock_id: lock_id + * @param [in] timeout_ms: wait time, unit: ms. If timeout_ms is greater than 0, wait for the specified time. + * If timeout_ms is equal to 0, do not wait. If timeout_ms is less than 0, wait forever. + * @return 0 success + * @return != 0 fail + */ +int cm_dl_lock(unsigned int lock_id, int timeout_ms); + +/** + * Unlock. + * @param [in] lock_id: lock_id + * @return 0 success + * @return != 0 fail + */ +int cm_dl_unlock(unsigned int lock_id); + +/** + * Clean the lock of other instance. + * @param [in] lock_id: lock_id + * @param [in] inst_id: target inst_id + * @return 0 success + * @return != 0 fail + */ +int cm_dl_clean(unsigned int lock_id, unsigned long long inst_id); + +/** + * Get the lock's owner. + * @param [in] lock_id: lock_id + * @param [out] inst_id: owner's inst_id. If the lock does not have an owner, CM_INVALID_INST_ID will be returned. + * @return 0 success + * @return != 0 fail + */ +int cm_dl_getowner(unsigned int lock_id, unsigned long long *inst_id); + +/** + * Get the lock's time. + * @param [in] lock_id: lock_id + * @param [out] locktime: locktime of the lock. If the lock does not have an owner, 0 will be returned. + * @return 0 success + * @return != 0 fail + */ +int cm_dl_getlocktime(unsigned int lock_id, unsigned long long *locktime); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/pkg/src/cms/cbb/test_log.c b/pkg/src/cms/cbb/test_log.c new file mode 100644 index 0000000000000000000000000000000000000000..6a464ee055b9c1f3d83d3b1efc164abeec136299 --- /dev/null +++ b/pkg/src/cms/cbb/test_log.c @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include "test_log.h" + +#ifdef TEST_LOG_ENABLED +#define LOG_FILE "/opt/cantian/dss/test.log" +static pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER; + +void get_current_time(char *buffer, size_t size) { + time_t raw_time; + struct tm *time_info; + + time(&raw_time); + time_info = localtime(&raw_time); + strftime(buffer, size, "%Y-%m-%d %H:%M:%S", time_info); +} +#endif + +void write_log_to_file(const char *file, int line, const char *format, ...) { +#ifdef TEST_LOG_ENABLED + pthread_mutex_lock(&log_mutex); + + FILE *log_file = fopen(LOG_FILE, "a"); + if (log_file != NULL) { + char time_str[20]; + get_current_time(time_str, sizeof(time_str)); + + fprintf(log_file, "[%s] (%s:%d) ", time_str, file, line); + + va_list args; + va_start(args, format); + vfprintf(log_file, format, args); + va_end(args); + + fprintf(log_file, "\n"); + fclose(log_file); + } else { + fprintf(stderr, "Error opening log file: %s\n", LOG_FILE); + } + + pthread_mutex_unlock(&log_mutex); +#endif +} \ No newline at end of file diff --git a/pkg/src/cms/cbb/test_log.h b/pkg/src/cms/cbb/test_log.h new file mode 100644 index 0000000000000000000000000000000000000000..969bad5d20af3fb51f619794c6936f30ccc0f703 --- /dev/null +++ b/pkg/src/cms/cbb/test_log.h @@ -0,0 +1,11 @@ +#ifndef TEST_LOG_H +#define TEST_LOG_H + +#define LOG(format, ...) write_log_to_file(__FILE__, __LINE__, format, ##__VA_ARGS__) + +#ifdef TEST_LOG_ENABLED +void get_current_time(char *buffer, size_t size); +#endif +void write_log_to_file(const char *file, int line, const char *format, ...); + +#endif \ No newline at end of file diff --git a/pkg/src/cms/cms/cms_detect_error.c b/pkg/src/cms/cms/cms_detect_error.c index 1598d03e9992a97beedade828e2423b104d8321a..0013976400e226a8e6fe37b7d9f2b4b8262825d3 100644 --- a/pkg/src/cms/cms/cms_detect_error.c +++ b/pkg/src/cms/cms/cms_detect_error.c @@ -72,7 +72,7 @@ status_t cms_get_script_from_memory(cms_res_t *res) status_t cms_exec_script_inner(cms_res_t res, char *type) { status_t result = CT_ERROR; - status_t ret = cms_exec_res_script(res.script, type, res.check_timeout, &result); + status_t ret = cms_exec_res_script(res.script, type, g_cms_param->node_id, res.check_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_DEBUG_INF("script executed successfully, script=%s, type=%s", res.script, type); @@ -246,7 +246,7 @@ status_t cms_daemon_stop_pull(void) status_t result = CT_ERROR; status_t ret = CT_ERROR; CMS_LOG_INF("start exec stop rerun script, script=%s", g_cms_param->stop_rerun_script); - ret = cms_exec_res_script(g_cms_param->stop_rerun_script, "disable", CMS_STOP_RERUN_SCRIPT_TIMEOUT, &result); + ret = cms_exec_res_script(g_cms_param->stop_rerun_script, "disable", g_cms_param->node_id, CMS_STOP_RERUN_SCRIPT_TIMEOUT, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec stop rerun script succeed, script=%s", g_cms_param->stop_rerun_script); diff --git a/pkg/src/cms/cms/cms_disk_lock.c b/pkg/src/cms/cms/cms_disk_lock.c index d375c6e00ec93329a1eea24e0fb3c9092a3a8953..15aca3071ce9b0f5c07c4b20c266f0b6e3df820c 100644 --- a/pkg/src/cms/cms/cms_disk_lock.c +++ b/pkg/src/cms/cms/cms_disk_lock.c @@ -322,7 +322,7 @@ static status_t cms_disk_lock_try_lock_sd(cms_disk_lock_t* lock) time_t lock_time = LOCKR_LOCK_TIME(lock->dlock); time_t now_time = time(NULL); time_t diff_time = now_time - lock_time; - if (diff_time <= CMS_DISK_LOCK_TIMEOUT) { + if (diff_time <= CMS_DISK_LOCK_SD_RELEASE_TIME) { return CT_EAGAIN; } @@ -416,7 +416,7 @@ void cms_exec_exit_proc(void) cms_inc_exit_num(res); } else { CMS_LOG_INF("exit_num file does not exist"); - cms_exec_res_script(res.script, "-inc_exit_num", res.check_timeout, &result); + cms_exec_res_script(res.script, "-inc_exit_num", g_cms_param->node_id, res.check_timeout, &result); cm_spin_unlock(&g_exit_num_lock); cms_kill_self_by_exit(); } diff --git a/pkg/src/cms/cms/cms_disk_lock.h b/pkg/src/cms/cms/cms_disk_lock.h index 16700505ad211aeb741976f7ac79fb9ee19b2de6..b305c7dee3c0594749059a48708e3a40540ff029 100644 --- a/pkg/src/cms/cms/cms_disk_lock.h +++ b/pkg/src/cms/cms/cms_disk_lock.h @@ -96,6 +96,7 @@ typedef union u_cms_master_info_t { #define CMS_STAT_LOCK_MAGIC (*((uint64*)"CMS_LOCK")) #define CMS_DISK_LOCK_TIMEOUT 10 +#define CMS_DISK_LOCK_SD_RELEASE_TIME 4 #define CMS_EXIT_NUM 128 #define CMS_EXIT_COUNT_MAX 20 diff --git a/pkg/src/cms/cms/cms_gcc.c b/pkg/src/cms/cms/cms_gcc.c index 025dd2d25269845093daba4e9889c0eceefdd0b3..f87cb94e2bf5e9e71e49a02687016ec75b9e3bb2 100644 --- a/pkg/src/cms/cms/cms_gcc.c +++ b/pkg/src/cms/cms/cms_gcc.c @@ -1159,13 +1159,10 @@ const cms_res_t* cms_find_res_type(const cms_gcc_t* gcc, const char* res_type) if (gcc->res[res_id].magic == CMS_GCC_RES_MAGIC && cm_strcmpi(gcc->res[res_id].type, res_type) == 0) { res = &gcc->res[res_id]; - break; + return res; } - CMS_LOG_WAR("res type is not found, res_id = %u, res[res_id].magic = %llu, CMS_GCC_RES_MAGIC = " - "%llu,res[res_id].type = %s, ", - res_id, gcc->res[res_id].magic, CMS_GCC_RES_MAGIC, gcc->res[res_id].type); } - + CMS_LOG_WAR("res type [%s] is not found", res_type); return res; } diff --git a/pkg/src/cms/cms/cms_gcc_exp.c b/pkg/src/cms/cms/cms_gcc_exp.c index aa10bc66d05772adacadeb66b42b1f206f7c7e10..7f7400b5f517effa4019a9b06d52ca4300c4704f 100644 --- a/pkg/src/cms/cms/cms_gcc_exp.c +++ b/pkg/src/cms/cms/cms_gcc_exp.c @@ -793,11 +793,15 @@ status_t cms_keep_recent_files_remote(const char *bak_path, char *prefix) CMS_LOG_ERR("get current work directory failed, error code %d.", errno); return CT_ERROR; } + + if (g_cms_param->gcc_type == CMS_DEV_TYPE_SD) { + return CT_SUCCESS; + } - if (g_cms_param->gcc_type != CMS_DEV_TYPE_DBS) { - ret = cms_remove_old_files(dirname, prefix); - } else { + if (g_cms_param->gcc_type == CMS_DEV_TYPE_DBS) { ret = cms_remove_old_files_dbs(dirname, prefix); + } else { + ret = cms_remove_old_files(dirname, prefix); } if (ret != CT_SUCCESS) { @@ -853,6 +857,10 @@ status_t cms_create_gcc_backup_files_remote(date_t bak_time, const char *bak_typ home_path, bak_type, time_str); PRTS_RETURN_IFERR(ret); + if (g_cms_param->gcc_type == CMS_DEV_TYPE_SD) { + return CT_SUCCESS; + } + if (g_cms_param->gcc_type != CMS_DEV_TYPE_DBS) { ret = snprintf_s(dir_name, CMS_FILE_NAME_BUFFER_SIZE, CMS_MAX_FILE_NAME_LEN, "%s/gcc_backup/", home_path); PRTS_RETURN_IFERR(ret); @@ -873,17 +881,17 @@ status_t cms_create_gcc_backup_files_remote(date_t bak_time, const char *bak_typ home_path, bak_type, time_str); PRTS_RETURN_IFERR(ret); - if (g_cms_param->gcc_type != CMS_DEV_TYPE_DBS) { - CT_RETURN_IFERR(cms_backup_binary_gcc(file_name)); - } else { + if (g_cms_param->gcc_type == CMS_DEV_TYPE_DBS) { if (cm_get_dbs_last_file_handle(file_name, &gcc_file_handle)) { printf("Failed to get gcc backup dir handle\n"); return CT_ERROR; } CT_RETURN_IFERR(cms_backup_binary_gcc_dbs(&gcc_file_handle)); + return CT_SUCCESS; } + CT_RETURN_IFERR(cms_backup_binary_gcc(file_name)); return CT_SUCCESS; } diff --git a/pkg/src/cms/cms/cms_instance.c b/pkg/src/cms/cms/cms_instance.c index 796329f3e51e121c95e2f759c37391e298b1dc1d..941235e3da361e729e1d0f1c64e9fb31a3e9aa30 100644 --- a/pkg/src/cms/cms/cms_instance.c +++ b/pkg/src/cms/cms/cms_instance.c @@ -503,16 +503,25 @@ status_t cms_update_version(void) CMS_LOG_ERR("cms gcc version bigger than db version, cmp_result = %d.", cmp_result); return CT_ERROR; } - CM_FREE_PTR(resident_gcc); // 如果是fullstart，则更新gcc CT_RETURN_IFERR(cms_is_all_restart(&all_restart)); if (!all_restart) { CMS_LOG_INF("cms gcc version not need update."); + CM_FREE_PTR(resident_gcc); // 更新内存gcc (void)cms_notify_load_gcc(); return CT_SUCCESS; } + + if (resident_gcc->head.ver_main != 0) { + CMS_LOG_INF("cms gcc version no more need update."); + CM_FREE_PTR(resident_gcc); + // 更新内存gcc + (void)cms_notify_load_gcc(); + return CT_SUCCESS; + } + CM_FREE_PTR(resident_gcc); ret = cms_update_gcc_ver(cms_version.main, cms_version.major, cms_version.revision, cms_version.inner); if (ret != CT_SUCCESS) { diff --git a/pkg/src/cms/cms/cms_iofence.c b/pkg/src/cms/cms/cms_iofence.c index e3c36adf4602b97d2e94e724968eaacce2c88139..ed0af82331ad551b6acd3c2e2783cfc9cae42ebb 100644 --- a/pkg/src/cms/cms/cms_iofence.c +++ b/pkg/src/cms/cms/cms_iofence.c @@ -262,11 +262,35 @@ void try_cms_file_kick_node(uint32 node_id, uint32 res_id, iofence_type_t iofenc CM_ABORT_REASONABLE(0, "[CMS] ABORT INFO: cms exec iof error."); } +void try_cms_dss_kick_node(uint32 node_id, uint32 res_id, iofence_type_t iofence_type) +{ + status_t ret = CT_ERROR; + for (int i = 0; i < IOF_RETRY_NUM; i++) { + ret = cms_res_iofence(node_id, res_id); + if (ret == CT_SUCCESS) { + CMS_LOG_INF("kick node succ, node_id %u", node_id); + return; + } + cm_sleep(IOF_CHECK_INTERVAL); + CMS_LOG_ERR("file iof failed, ret %d, node_id %u", ret, node_id); + } + if (cms_daemon_stop_pull() != CT_SUCCESS) { + CMS_LOG_ERR("stop cms daemon process failed."); + } + CM_ABORT_REASONABLE(0, "[CMS] ABORT INFO: cms exec iof error."); +} + void try_cms_kick_node(uint32 node_id, uint32 res_id, iofence_type_t iofence_type) { if (cm_dbs_is_enable_dbs() == CT_TRUE) { try_cms_dbs_kick_node(node_id, res_id, iofence_type); - } else { - try_cms_file_kick_node(node_id, res_id, iofence_type); + return; + } + + if (g_cms_param->gcc_type == CMS_DEV_TYPE_SD) { + try_cms_dss_kick_node(node_id, res_id, iofence_type); + return; } + + try_cms_file_kick_node(node_id, res_id, iofence_type); } \ No newline at end of file diff --git a/pkg/src/cms/cms/cms_main.c b/pkg/src/cms/cms/cms_main.c index b0bcbcb1cb3f8f546ec49b442cfde52a2a020d25..57968e3c0a285b548a640eeceb25b760563b62a2 100644 --- a/pkg/src/cms/cms/cms_main.c +++ b/pkg/src/cms/cms/cms_main.c @@ -260,10 +260,7 @@ static void cms_info_log_print(cms_cmd_def_t* cmd_def) status_t cm_get_gcc_file_handle() { - if (g_cms_param->gcc_type == CMS_DEV_TYPE_DBS) { - // 去nas时cms tool命令没有初始化dbstor，tool命令内部有去加载dbstor，若g_cms_param->gcc_home文件不存在，正常报错即可。 - return CT_SUCCESS; - } else { + if (g_cms_param->gcc_type == CMS_DEV_TYPE_NFS) { if (cm_file_exist(g_cms_param->gcc_home) == CT_FALSE) { return CT_ERROR; } diff --git a/pkg/src/cms/cms/cms_param.c b/pkg/src/cms/cms/cms_param.c index 4d289e66a0f2447487b9d0e83c2b1c91ee6ebbdc..10cc53050a34faf2317b0f3a764f6374d94d5bb2 100644 --- a/pkg/src/cms/cms/cms_param.c +++ b/pkg/src/cms/cms/cms_param.c @@ -463,46 +463,50 @@ status_t cms_get_mes_ssl_config(config_t *cfg) status_t cms_get_dbstore_config_value(config_t *cfg) { - char* value; + char* use_dbs_value; char* gcc_type; status_t ret; - bool32 enable = CT_FALSE; + char* namespace_value = NULL; // dataPgSize is not used in the cms uint32 dataPgSize = CT_MAX_UINT32; - value = cm_get_config_value(cfg, "_USE_DBSTOR"); + use_dbs_value = cm_get_config_value(cfg, "_USE_DBSTOR"); gcc_type = cm_get_config_value(cfg, "GCC_TYPE"); - if (cm_strcmpi(gcc_type, "FILE") == 0 && cm_strcmpi(value, "FALSE") == 0) { - enable = CT_FALSE; - } else if (cm_strcmpi(gcc_type, "DBS") == 0 && (value == NULL || cm_strcmpi(value, "TRUE") == 0)) { - enable = CT_TRUE; - CMS_LOG_INF("DBStore not enabled for DBS"); - cms_set_recv_timeout(); - g_cms_dbstor_enable = CT_TRUE; - } else if (cm_strcmpi(gcc_type, "NFS") == 0 && cm_strcmpi(value, "TRUE") == 0) { - enable = CT_TRUE; - CMS_LOG_INF("DBStore not enabled for NFS"); - cms_set_recv_timeout(); - g_cms_dbstor_enable = CT_TRUE; - } else if (cm_strcmpi(gcc_type, "NFS") == 0 && cm_strcmpi(value, "FALSE") == 0) { - enable = CT_FALSE; + if (cm_strcmpi(gcc_type, "FILE") == 0 && cm_strcmpi(use_dbs_value, "FALSE") == 0) { + CMS_LOG_INF("DBStore disabled for FILE"); + ret = cm_dbs_set_cfg(CT_FALSE, dataPgSize, CT_DFLT_CTRL_BLOCK_SIZE, namespace_value, 0, CT_FALSE, 0); + return ret; + } + if (cm_strcmpi(gcc_type, "NFS") == 0 && cm_strcmpi(use_dbs_value, "FALSE") == 0) { CMS_LOG_INF("DBStore disabled for NFS"); - } else { - CMS_LOG_ERR("Invalid parameters for '_USE_DBSTOR': gcc_type=%s, value=%s", gcc_type, value); - return CT_ERROR; + ret = cm_dbs_set_cfg(CT_FALSE, dataPgSize, CT_DFLT_CTRL_BLOCK_SIZE, namespace_value, 0, CT_FALSE, 0); + return ret; } - value = cm_get_config_value(cfg, "_DBSTOR_NAMESPACE"); - if (value == NULL) { + if (cm_strcmpi(gcc_type, "SD") == 0 && cm_strcmpi(use_dbs_value, "FALSE") == 0) { + CMS_LOG_INF("DBStore disabled for SD"); + ret = cm_dbs_set_cfg(CT_FALSE, dataPgSize, CT_DFLT_CTRL_BLOCK_SIZE, namespace_value, 0, CT_FALSE, 0); + return ret; + } + + namespace_value = cm_get_config_value(cfg, "_DBSTOR_NAMESPACE"); + if (namespace_value == NULL) { CMS_LOG_ERR("invalid parameter value of '_DBSTOR_NAMESPACE'"); return CT_ERROR; } - ret = cm_dbs_set_cfg(enable, dataPgSize, CT_DFLT_CTRL_BLOCK_SIZE, value, 0, CT_FALSE, 0); - if (ret != CT_SUCCESS) { - CMS_LOG_ERR("cms set dbstore config failed"); - return CT_ERROR; + if (cm_strcmpi(gcc_type, "DBS") == 0 && (use_dbs_value == NULL || cm_strcmpi(use_dbs_value, "TRUE") == 0)) { + CMS_LOG_INF("Configuring DBStore for DBS"); + cms_set_recv_timeout(); + + ret = cm_dbs_set_cfg(CT_TRUE, dataPgSize, CT_DFLT_CTRL_BLOCK_SIZE, namespace_value, 0, CT_FALSE, 0); + if (ret != CT_SUCCESS) { + CMS_LOG_ERR("cms set dbstore config failed"); + return CT_ERROR; + } + return CT_SUCCESS; } - return CT_SUCCESS; + CMS_LOG_ERR("Invalid parameters for '_USE_DBSTOR': gcc_type=%s, value=%s", gcc_type, use_dbs_value); + return CT_ERROR; } status_t cms_load_param(int64* time_stamp) @@ -712,14 +716,12 @@ status_t cms_load_param(int64* time_stamp) CMS_LOG_INF("DBStore not enabled for DBS"); cms_set_recv_timeout(); g_cms_dbstor_enable = CT_TRUE; - } else if (cm_strcmpi(gcc_type, "NFS") == 0 && cm_strcmpi(value, "TRUE") == 0) { - enable = CT_TRUE; - CMS_LOG_INF("DBStore not enabled for NFS"); - cms_set_recv_timeout(); - g_cms_dbstor_enable = CT_TRUE; } else if (cm_strcmpi(gcc_type, "NFS") == 0 && cm_strcmpi(value, "FALSE") == 0) { enable = CT_FALSE; CMS_LOG_INF("DBStore disabled for NFS"); + } else if (cm_strcmpi(gcc_type, "SD") == 0 && cm_strcmpi(value, "FALSE") == 0) { + enable = CT_FALSE; + CMS_LOG_INF("DBStore not enabled for SD"); } else { CMS_LOG_ERR("Invalid parameters for '_USE_DBSTOR': gcc_type=%s, value=%s", gcc_type, value); return CT_ERROR; diff --git a/pkg/src/cms/cms/cms_stat.c b/pkg/src/cms/cms/cms_stat.c index aaf6dfb3e98fe213bb5f3716597f8a7bf5997bd3..25d52c42b035b46ba6c37675060cd4bf57d60961 100644 --- a/pkg/src/cms/cms/cms_stat.c +++ b/pkg/src/cms/cms/cms_stat.c @@ -417,6 +417,7 @@ status_t cms_res_lock_init(void) CMS_RES_STAT_LOCK_POS(node_id, res_id), CMS_RLOCK_RES_STAT_LOCK_START(node_id, res_id), CMS_RLOCK_RES_STAT_LOCK_LEN, g_cms_param->node_id, &g_cms_inst->res_stat_lock[node_id][res_id], NULL, 0, CT_FALSE)); + CMS_LOG_INF("cms init stat lock node_id:%d, res_id:%d, offset:%lu", node_id, res_id, CMS_RES_STAT_LOCK_POS(node_id, res_id)); cms_disk_unlock(&g_cms_inst->res_stat_lock[node_id][res_id], DISK_LOCK_READ); } cm_init_thread_lock(&g_node_lock[node_id]); @@ -434,6 +435,7 @@ status_t cms_res_lock_init(void) CMS_RES_DATA_LOCK_POS(res_id, slot_id), CMS_RLOCK_RES_DATA_LOCK_START(res_id, slot_id), CMS_RLOCK_RES_DATA_LOCK_LEN, g_cms_param->node_id, &g_cms_inst->res_data_lock[res_id][slot_id], res_is_active, 0, CT_FALSE)); + CMS_LOG_INF("cms init data lock res_id:%d, slot_id:%d, offset:%lu", res_id, slot_id, CMS_RES_DATA_LOCK_POS(res_id, slot_id)); g_cms_inst->res_data_lock[res_id][slot_id].int64_param1 = res_id; cms_disk_unlock(&g_cms_inst->res_data_lock[res_id][slot_id], DISK_LOCK_READ); } @@ -538,9 +540,9 @@ void cms_exec_res_script_print_log(const char* arg, char *cmd) } } -status_t cms_exec_res_script(const char* script, const char* arg, uint32 timeout_ms, status_t* result) +status_t cms_exec_res_script(const char* script, const char* arg, uint32 node_id, uint32 timeout_ms, status_t* result) { - CMS_LOG_INF("begin cms exec res script."); + CMS_LOG_TIMER("begin cms exec res script."); char cmd[CMS_CMD_BUFFER_SIZE] = {0}; *result = CT_ERROR; errno_t ret = EOK; @@ -554,8 +556,8 @@ status_t cms_exec_res_script(const char* script, const char* arg, uint32 timeout (float)timeout_ms / CMS_TRANS_MS_TO_SECOND_FLOAT, script, arg); } else { ret = sprintf_s(cmd, CMS_CMD_BUFFER_SIZE, - "echo 'script begin';timeout %.2f %s %s %d;echo $?;echo 'script end\n';", - (float)timeout_ms / CMS_TRANS_MS_TO_SECOND_FLOAT, script, arg, (int32)g_cms_param->node_id); + "echo 'script begin';timeout %.2f %s %s %u;echo $?;echo 'script end\n';", + (float)timeout_ms / CMS_TRANS_MS_TO_SECOND_FLOAT, script, arg, node_id); } PRTS_RETURN_IFERR(ret); FILE* fp = popen(cmd, "r"); @@ -577,22 +579,23 @@ status_t cms_exec_res_script(const char* script, const char* arg, uint32 timeout } cmd_out[size] = 0; - CMS_LOG_INF("end cms exec res script."); - if (strstr(cmd_out, "RES_SUCCESS") != NULL) { - *result = CT_SUCCESS; - return CT_SUCCESS; - } + CMS_LOG_TIMER("end cms exec res script."); if (strstr(cmd_out, CMS_TIMEOUT_ERROR_NUMBER) != NULL) { *result = CT_TIMEDOUT; return CT_SUCCESS; } - if (strstr(cmd_out, "RES_MULTI") != NULL) { + if (strstr(cmd_out, "RES_EAGAIN") != NULL) { *result = CT_EAGAIN; return CT_SUCCESS; } + if (strstr(cmd_out, "RES_SUCCESS") != NULL) { + *result = CT_SUCCESS; + return CT_SUCCESS; + } + *result = CT_ERROR; return CT_SUCCESS; } @@ -612,7 +615,7 @@ status_t cms_res_init(uint32 res_id, uint32 timeout_ms) return CT_ERROR; } - ret = cms_exec_res_script(res.script, "-init", res.start_timeout, &result); + ret = cms_exec_res_script(res.script, "-init", g_cms_param->node_id, res.start_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec init script succeed, script=%s, res_id=%u", res.script, res_id); @@ -908,6 +911,65 @@ status_t cms_check_res_running(uint32 res_id) return CT_SUCCESS; } +status_t cms_check_dss_stat(cms_res_t res, cms_res_stat_t stat) +{ + status_t dss_status; + status_t ret; + + ret = cms_res_check(res.res_id, &dss_status); + if (ret != CT_SUCCESS || dss_status == CT_ERROR) { + CMS_LOG_ERR_LIMIT(LOG_PRINT_INTERVAL_SECOND_20, + "check dss failed, res_id=%u, script=%s, ret=%d, dss_status=%d", res.res_id, res.script, ret, dss_status); + return CT_ERROR; + } + + if (dss_status == CT_TIMEDOUT) { + CMS_LOG_ERR_LIMIT(LOG_PRINT_INTERVAL_SECOND_20, + "check dss stat timeout, res_id=%u, script=%s", res.res_id, res.script); + return CT_ERROR; + } + + if (dss_status == CT_EAGAIN) { + CMS_LOG_ERR_LIMIT(LOG_PRINT_INTERVAL_SECOND_20, + "dss work stat is abnormal, res_id=%u, script=%s", res.res_id, res.script); + return CT_ERROR; + } + cms_res_hb(res.res_id); + return CT_SUCCESS; +} + +status_t cms_res_register(uint32 instance_id, uint32 res_id) +{ + CMS_LOG_INF("begin exec res register, res_id=%u", res_id); + status_t ret; + status_t result; + cms_res_t res; + + CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); + if (cm_strcmpi(res.name, CMS_RES_TYPE_DSS) != 0) { + if (cms_get_res_id_by_type(CMS_RES_TYPE_DSS, &res_id) != CT_SUCCESS) { + CMS_LOG_ERR("failed to get dss res id."); + return CT_ERROR; + } + CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); + CMS_LOG_INF("Executing DSS script: %s for register db.", res.script); + } + + ret = cms_exec_res_script(res.script, "-reg", instance_id, res.stop_timeout, &result); + if (ret == CT_SUCCESS) { + if (result == CT_SUCCESS) { + CMS_LOG_INF("exec register script succeed, script=%s, instance_id=%u", res.script, instance_id); + } else { + CMS_LOG_ERR("exec register script succeed, but result is failed, script=%s, instance_id=%u", res.script, instance_id); + return result; + } + } else { + CMS_LOG_ERR("exec register script failed, script=%s", res.script); + } + CMS_LOG_INF("end exec res register succeed, instance_id=%u, ret=%d", instance_id, ret); + return CT_SUCCESS; +} + status_t cms_res_start(uint32 res_id, uint32 timeout_ms) { status_t ret; @@ -918,6 +980,29 @@ status_t cms_res_start(uint32 res_id, uint32 timeout_ms) CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); CMS_LOG_INF("begin start res, res_id=%u", res_id); + if (g_cms_param->gcc_type == CMS_DEV_TYPE_SD && cm_strcmpi(res.name, CMS_RES_TYPE_DB) == 0) { + CMS_LOG_INF("cms check dss stat before start db, res_id=%u", res_id); + cms_res_t dss_res; + uint32 dss_res_id; + cms_res_stat_t dss_res_stat; + // find dss resource + if (cms_get_res_id_by_type(CMS_RES_TYPE_DSS, &dss_res_id) != CT_SUCCESS) { + CMS_LOG_ERR("cms get res id failed, res_type %s", CMS_RES_TYPE_DSS); + return CT_ERROR; + } + + CT_RETURN_IFERR(cms_get_res_by_id(dss_res_id, &dss_res)); + get_cur_res_stat(dss_res_id, &dss_res_stat); + if (cms_check_dss_stat(dss_res, dss_res_stat) != CT_SUCCESS) { + CMS_LOG_ERR("DSS status is abnormal, unable to start Cantian."); + return CT_ERROR; + } + ret = cms_res_register(g_cms_param->node_id, dss_res_id); + if (ret != CT_SUCCESS) { + CMS_LOG_ERR("DSS register failed, unable to start Cantian."); + return CT_ERROR; + } + } CMS_LOG_INF("begin to get start lock, res_id=%u", res_id); if (cms_get_res_start_lock(res_id) != CT_SUCCESS) { @@ -932,7 +1017,9 @@ status_t cms_res_start(uint32 res_id, uint32 timeout_ms) return CT_SUCCESS; } - if (wait_for_cluster_reform_done(res_id) != CT_SUCCESS) { + if (cm_strcmpi(res.name, CMS_RES_TYPE_DSS) == 0) { + CMS_LOG_INF("CMS does not need to wait for reform when starting DSS"); + } else if (wait_for_cluster_reform_done(res_id) != CT_SUCCESS) { CMS_LOG_ERR("cms wait cluster reform done failed"); cms_release_res_start_lock(res_id); return CT_ERROR; @@ -944,7 +1031,7 @@ status_t cms_res_start(uint32 res_id, uint32 timeout_ms) } CMS_LOG_INF("begin to exec start script, res_id=%u", res_id); - ret = cms_exec_res_script(res.script, "-start", res.start_timeout, &result); + ret = cms_exec_res_script(res.script, "-start", g_cms_param->node_id, res.start_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec start script succeed, script=%s, res_id=%u", res.script, res_id); @@ -974,7 +1061,7 @@ status_t cms_res_stop_by_force(uint32 res_id, uint8 need_write_disk) CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); - ret = cms_exec_res_script(res.script, "-stop_force", res.stop_timeout, &result); + ret = cms_exec_res_script(res.script, "-stop_force", g_cms_param->node_id, res.stop_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec stop by force script succeed, script=%s, res_id=%u", res.script, res_id); @@ -1001,7 +1088,7 @@ status_t cms_res_stop(uint32 res_id, uint8 need_write_disk) CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); - ret = cms_exec_res_script(res.script, "-stop", res.stop_timeout, &result); + ret = cms_exec_res_script(res.script, "-stop", g_cms_param->node_id, res.stop_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec stop script succeed, script=%s, res_id=%u", res.script, res_id); @@ -1019,6 +1106,38 @@ status_t cms_res_stop(uint32 res_id, uint8 need_write_disk) return ret; } +status_t cms_res_iofence(uint32 instance_id, uint32 res_id) +{ + CMS_LOG_INF("begin exec res iofence, res_id=%u", res_id); + status_t ret; + status_t result; + cms_res_t res; + + CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); + if (cm_strcmpi(res.name, CMS_RES_TYPE_DSS) != 0) { + if (cms_get_res_id_by_type(CMS_RES_TYPE_DSS, &res_id) != CT_SUCCESS) { + CMS_LOG_ERR("failed to get dss res id."); + return CT_ERROR; + } + CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); + CMS_LOG_INF("Executing DSS script: %s for iofence db.", res.script); + } + + ret = cms_exec_res_script(res.script, "-kick", instance_id, res.stop_timeout, &result); + if (ret == CT_SUCCESS) { + if (result == CT_SUCCESS) { + CMS_LOG_INF("exec kick script succeed, script=%s, instance_id=%u", res.script, instance_id); + } else { + CMS_LOG_ERR("exec kick script succeed, but result is failed, script=%s, instance_id=%u", res.script, instance_id); + return result; + } + } else { + CMS_LOG_ERR("exec kick script failed, script=%s", res.script); + } + CMS_LOG_INF("end exec res kick succeed, instance_id=%u, ret=%d", instance_id, ret); + return CT_SUCCESS; +} + status_t cms_res_check(uint32 res_id, status_t *res_status) { status_t ret = CT_SUCCESS; @@ -1033,7 +1152,7 @@ status_t cms_res_check(uint32 res_id, status_t *res_status) return CT_SUCCESS; } - ret = cms_exec_res_script(res.script, "-check", res.check_timeout, res_status); + ret = cms_exec_res_script(res.script, "-check", g_cms_param->node_id, res.check_timeout, res_status); cm_atomic32_dec(&res_stat->checking); if (ret == CT_SUCCESS) { @@ -1058,7 +1177,7 @@ status_t cms_res_reset(uint32 res_id) CT_RETURN_IFERR(cms_get_res_by_id(res_id, &res)); - ret = cms_exec_res_script(res.script, "-reset", res.stop_timeout, &result); + ret = cms_exec_res_script(res.script, "-reset", g_cms_param->node_id, res.stop_timeout, &result); if (ret == CT_SUCCESS) { if (result == CT_SUCCESS) { CMS_LOG_INF("exec reset script succeed, res id %u", res_id); @@ -1152,6 +1271,11 @@ status_t cms_res_detect_online(uint32 res_id, cms_res_stat_t *old_stat) cm_thread_unlock(&g_res_session[res_id].lock); return CT_ERROR; } + res_stat->last_stat_change = cm_now(); + res_stat->work_stat = 1; + res_stat->hb_time = cm_now(); + res_stat->session_id = res_id; + res_stat->inst_id = 0; cms_stat_set(res_stat, CMS_RES_ONLINE, &is_changed); if (!(is_changed)) { cm_thread_unlock(&g_res_session[res_id].lock); diff --git a/pkg/src/cms/cms/cms_stat.h b/pkg/src/cms/cms/cms_stat.h index c358f96e4416017a590bebbc8c0f93e231970604..c9963705d5f2781820af27623df37e7f0ed06f1c 100644 --- a/pkg/src/cms/cms/cms_stat.h +++ b/pkg/src/cms/cms/cms_stat.h @@ -270,6 +270,7 @@ status_t cms_res_init(uint32 res_id, uint32 timeout_ms); status_t cms_res_start(uint32 res_id, uint32 timeout_ms); status_t cms_res_stop(uint32 res_id, uint8 need_write_disk); status_t cms_res_stop_by_force(uint32 res_id, uint8 need_write_disk); +status_t cms_res_iofence(uint32 node_id, uint32 res_id); status_t cms_res_check(uint32 res_id, status_t *res_status); status_t cms_res_reset(uint32 res_id); @@ -313,7 +314,7 @@ status_t cms_stat_write_to_disk(uint32 node_id, uint32 res_id, cms_res_stat_t* s void cms_stat_set(cms_res_stat_t* res_stat, cms_stat_t new_stat, bool32* isChanged); status_t inc_stat_version(void); status_t cms_vote_disk_init(void); -status_t cms_exec_res_script(const char* script, const char* arg, uint32 timeout_ms, status_t* result); +status_t cms_exec_res_script(const char* script, const char* arg, uint32 node_id, uint32 timeout_ms, status_t* result); status_t cms_aync_update_res_hb(cms_res_stat_t *res_stat_disk, uint32 res_id); status_t cms_update_disk_hb(void); status_t cms_is_all_restart(bool32 *all_restart); @@ -333,6 +334,7 @@ status_t cms_server_stat(uint32 node_id, bool32* cms_online); status_t cms_get_node_view(uint64* cms_online_bitmap); status_t cms_check_res_running(uint32 res_id); +status_t cms_check_dss_stat(cms_res_t res, cms_res_stat_t stat); status_t cms_init_mes_channel_version(void); status_t cms_get_mes_channel_version(uint64* version); status_t cms_get_cluster_res_list_4tool(uint32 res_id, cms_tool_res_stat_list_t *res_list); diff --git a/pkg/src/cms/cms/cms_work.c b/pkg/src/cms/cms/cms_work.c index fe09e1a43609ff8ddf996cd4ac3fdc75eacfd1bb..8acb0967d67261f34c287d23c5f9b11ad22c65e7 100644 --- a/pkg/src/cms/cms/cms_work.c +++ b/pkg/src/cms/cms/cms_work.c @@ -245,8 +245,17 @@ status_t cms_start_res_cluster(cms_packet_head_t* msg, char* err_info, uint32 er return CT_SUCCESS; } -status_t cms_create_res_disable() +status_t cms_create_res_disable(uint32 res_id) { + cms_res_t res; + if (cms_get_res_by_id(res_id, &res) != CT_SUCCESS) { + return CT_ERROR; + } + + if (cm_strcmpi(res.name, CMS_RES_TYPE_DB) != 0) { + return CT_SUCCESS; + } + int32 handle = CT_INVALID_HANDLE; char res_disable_file[CMS_PATH_BUFFER_SIZE] = { 0 }; errno_t err = sprintf_s(res_disable_file, CMS_PATH_BUFFER_SIZE, "%s/%s", g_cms_param->cms_home, "res_disable"); @@ -260,8 +269,16 @@ status_t cms_create_res_disable() return CT_SUCCESS; } -status_t cms_remove_res_disable() +status_t cms_remove_res_disable(uint32 res_id) { + cms_res_t res; + if (cms_get_res_by_id(res_id, &res) != CT_SUCCESS) { + return CT_ERROR; + } + + if (cm_strcmpi(res.name, CMS_RES_TYPE_DB) != 0) { + return CT_SUCCESS; + } char res_disable_file[CMS_PATH_BUFFER_SIZE] = { 0 }; errno_t err = sprintf_s(res_disable_file, CMS_PATH_BUFFER_SIZE, "%s/%s", g_cms_param->cms_home, "res_disable"); PRTS_RETURN_IFERR(err); @@ -285,7 +302,7 @@ status_t cms_start_res_local(uint32 res_id, uint32 timeout, char* err_info) return ret; } - ret = cms_remove_res_disable(); + ret = cms_remove_res_disable(res_id); if (ret != CT_SUCCESS) { err = strcpy_s(err_info, CMS_INFO_BUFFER_SIZE, "remove res disable file failed"); cms_securec_check(err); @@ -532,7 +549,7 @@ status_t cms_stop_res_local_force(uint32 res_id, char* err_info) return ret; } - ret = cms_create_res_disable(); + ret = cms_create_res_disable(res_id); if (ret != CT_SUCCESS) { err = strcpy_s(err_info, CMS_INFO_BUFFER_SIZE, "create res disable file failed"); cms_securec_check(err); @@ -561,7 +578,7 @@ status_t cms_check_res_status(uint32 res_id, char* err_info) } if (res_status == CT_EAGAIN) { - err = strcpy_s(err_info, CMS_INFO_BUFFER_SIZE, "More than one resource is running."); + err = strcpy_s(err_info, CMS_INFO_BUFFER_SIZE, "resource work stat is abnormal"); cms_securec_check(err); return CT_ERROR; } @@ -601,7 +618,7 @@ status_t cms_stop_res_local(uint32 res_id, char* err_info) return ret; } - ret = cms_create_res_disable(); + ret = cms_create_res_disable(res_id); if (ret != CT_SUCCESS) { err = strcpy_s(err_info, CMS_INFO_BUFFER_SIZE, "create res disable file failed"); cms_securec_check(err); @@ -3869,8 +3886,11 @@ void cms_res_check_timer_entry(thread_t* thread) } min_interval = MIN(res.check_interval, min_interval); min_interval = MAX(max_interval, min_interval); - get_cur_res_stat(i, &stat); + if (cm_strcmpi(res.name, CMS_RES_TYPE_DSS) == 0 && cms_check_dss_stat(res, stat) == CT_SUCCESS) { + cms_res_detect_online(i, &stat); + continue; + } now_time = cm_now(); // cluster time cms_detect_osclock_abnormal(now_time, last_refresh_time); last_refresh_time = now_time; diff --git a/pkg/src/cms/interface/cms_interface.h b/pkg/src/cms/interface/cms_interface.h index 3aeb5e928d32012fbaddc654d7182427f19bdf9c..6f485a81e5270fa1aa027182f7622bc395241fa8 100644 --- a/pkg/src/cms/interface/cms_interface.h +++ b/pkg/src/cms/interface/cms_interface.h @@ -39,6 +39,7 @@ extern "C" #define CMS_CFG_FILENAME "cms.ini" #define CMS_RES_TYPE_CTSTORE "CTSTORE" #define CMS_RES_TYPE_DB "DB" +#define CMS_RES_TYPE_DSS "DSS" #define CMS_MAX_RES_SLOT_COUNT 8 #define CMS_CLI_HB_INTERVAL (MICROSECS_PER_SECOND * 2) #define CMS_CLI_UDS_SEND_TMOUT 1000 diff --git a/pkg/src/common/cm_defs.h b/pkg/src/common/cm_defs.h index 7382464fce90516da54cd979798ecdd3abf3066f..8797883a68e802dd0fecd8c93c87e409108dff84 100644 --- a/pkg/src/common/cm_defs.h +++ b/pkg/src/common/cm_defs.h @@ -1245,13 +1245,22 @@ typedef struct st_handle_mutiple_ptrs { } \ } while (0) +#define CT_PRINT_IFERR(ret, param, limit) \ + do { \ + status_t _status_ = (ret); \ + if (SECUREC_UNLIKELY(_status_ != CT_SUCCESS)) { \ + printf("Error,the parameter [%s] is too large,the limit is %d. \n", param, limit); \ + return _status_; \ + } \ + } while (0) + // print ERROR if error occurs -#define CT_PRINT_IFERR(ret,param,limit) \ - do { \ - status_t _status_ = (ret); \ - if (SECUREC_UNLIKELY(_status_ != CT_SUCCESS)) { \ - printf("Error,the parameter [%s] is too large,the limit is %d. \n",param,limit);\ - return _status_; \ +#define CT_PRINT_IFERR2(ret, param, limit) \ + do { \ + status_t _status_ = (ret); \ + if (SECUREC_UNLIKELY(_status_ == -1)) { \ + printf("Error,the parameter [%s] is too large,the limit is %d. \n", param, limit); \ + return _status_; \ } \ } while (0) diff --git a/pkg/src/common/cm_device.c b/pkg/src/common/cm_device.c index 57e0a8e3fb3002fddacd179c814d01e1049de8f0..980ff91d0f4d43fb4d27fdb8824e22c8e993ee49 100644 --- a/pkg/src/common/cm_device.c +++ b/pkg/src/common/cm_device.c @@ -44,6 +44,15 @@ raw_device_op_t g_raw_device_op; cm_check_file_error_t g_check_file_error = NULL; +#define CT_THROW_RAW_ERROR \ + do { \ + int32 errcode; \ + const char *errmsg = NULL; \ + g_raw_device_op.raw_get_error(&errcode, &errmsg); \ + CT_THROW_ERROR(ERR_DSS_FAILED, errcode, errmsg); \ + } while (0) + + void cm_raw_device_register(raw_device_op_t *device_op) { g_raw_device_op = *device_op; @@ -75,13 +84,13 @@ status_t cm_access_device(device_type_t type, const char *file_name, uint32 mode if (type == DEV_TYPE_FILE) { return cm_access_file(file_name, mode); } else if (type == DEV_TYPE_RAW) { - if (g_raw_device_op.raw_exist == NULL) { + if (g_raw_device_op.raw_stat == NULL) { CT_THROW_ERROR(ERR_DEVICE_NOT_SUPPORT); return CT_ERROR; } - bool32 result = CT_FALSE; - if (g_raw_device_op.raw_exist(file_name, &result) != CT_SUCCESS || !result) { + dss_stat_t item = {0}; + if (g_raw_device_op.raw_stat(file_name, &item) != CT_SUCCESS) { return CT_ERROR; } return CT_SUCCESS; @@ -122,8 +131,9 @@ status_t cm_create_device(const char *name, device_type_t type, uint32 flags, in timeval_t tv_begin; status_t ret = CT_SUCCESS; io_record_stat_t io_stat = IO_STAT_SUCCESS; + uint32 mode = O_BINARY | O_SYNC | O_RDWR | O_EXCL | flags; if (type == DEV_TYPE_FILE) { - if (cm_create_file(name, O_BINARY | O_SYNC | O_RDWR | O_EXCL | flags, handle) != CT_SUCCESS) { + if (cm_create_file(name, mode, handle) != CT_SUCCESS) { cm_check_file_error(); return CT_ERROR; } @@ -133,10 +143,10 @@ status_t cm_create_device(const char *name, device_type_t type, uint32 flags, in return CT_ERROR; } - if (g_raw_device_op.raw_create(name, O_BINARY | O_SYNC | O_RDWR | O_EXCL | flags) != CT_SUCCESS) { + if (g_raw_device_op.raw_create(name, mode) != CT_SUCCESS) { return CT_ERROR; } - if (g_raw_device_op.raw_open(name, flags, handle) != CT_SUCCESS) { + if (g_raw_device_op.raw_open(name, mode, handle) != CT_SUCCESS) { return CT_ERROR; } } else if (type == DEV_TYPE_ULOG) { @@ -317,12 +327,14 @@ status_t cm_open_device_common(const char *name, device_type_t type, uint32 flag return CT_SUCCESS; } -status_t cm_open_device(const char *name, device_type_t type, uint32 flags, int32 *handle){ - return cm_open_device_common(name, type ,flags, handle, CT_TRUE); +status_t cm_open_device(const char *name, device_type_t type, uint32 flags, int32 *handle) +{ + return cm_open_device_common(name, type, flags, handle, CT_TRUE); } -status_t cm_open_device_no_retry(const char *name, device_type_t type, uint32 flags, int32 *handle){ - return cm_open_device_common(name, type ,flags, handle, CT_FALSE); +status_t cm_open_device_no_retry(const char *name, device_type_t type, uint32 flags, int32 *handle) +{ + return cm_open_device_common(name, type, flags, handle, CT_FALSE); } void cm_close_device(device_type_t type, int32 *handle) @@ -332,7 +344,7 @@ void cm_close_device(device_type_t type, int32 *handle) cm_close_file(*handle); } else if (type == DEV_TYPE_RAW) { if (g_raw_device_op.raw_close != NULL) { - g_raw_device_op.raw_close(handle); + g_raw_device_op.raw_close(*handle); } } else if (type == DEV_TYPE_ULOG) { cantian_record_io_stat_begin(IO_RECORD_EVENT_NS_CLOSE_ULOG, &tv_begin); @@ -357,15 +369,12 @@ status_t cm_read_device(device_type_t type, int32 handle, int64 offset, void *bu return CT_ERROR; } } else if (type == DEV_TYPE_RAW) { - if (g_raw_device_op.raw_seek == NULL || g_raw_device_op.raw_read == NULL) { + if (g_raw_device_op.raw_pread == NULL) { CT_THROW_ERROR(ERR_DEVICE_NOT_SUPPORT); return CT_ERROR; } - if (g_raw_device_op.raw_seek(handle, offset, SEEK_SET) != offset) { - return CT_ERROR; - } - if (g_raw_device_op.raw_read(handle, offset, buf, size, &read_size) != CT_SUCCESS) { + if (g_raw_device_op.raw_pread(handle, buf, size, offset, &read_size) != CT_SUCCESS) { return CT_ERROR; } } else if (type == DEV_TYPE_ULOG) { @@ -420,7 +429,7 @@ status_t cm_read_device_nocheck(device_type_t type, int32 handle, int64 offset, CT_LOG_RUN_ERR("[cm_read_device] raw_seek handle %d offset %lld size %d.", handle, offset, size); return CT_ERROR; } - if (g_raw_device_op.raw_read(handle, offset, buf, size, &read_size) != CT_SUCCESS) { + if (g_raw_device_op.raw_read(handle, buf, size, &read_size) != CT_SUCCESS) { CT_LOG_RUN_ERR("[cm_read_device] raw_read handle %d offset %lld size %d.", handle, offset, size); return CT_ERROR; } @@ -505,15 +514,12 @@ status_t cm_write_device(device_type_t type, int32 handle, int64 offset, const v return CT_ERROR; } } else if (type == DEV_TYPE_RAW) { - if (g_raw_device_op.raw_seek == NULL || g_raw_device_op.raw_write == NULL) { + if (g_raw_device_op.raw_pwrite == NULL) { CT_THROW_ERROR(ERR_DEVICE_NOT_SUPPORT); return CT_ERROR; } - if (g_raw_device_op.raw_seek(handle, offset, SEEK_SET) != offset) { - return CT_ERROR; - } - if (g_raw_device_op.raw_write(handle, offset, buf, size) != CT_SUCCESS) { + if (g_raw_device_op.raw_pwrite(handle, buf, size, offset) != CT_SUCCESS) { return CT_ERROR; } } else if (type == DEV_TYPE_ULOG) { @@ -570,6 +576,11 @@ status_t cm_get_size_device(device_type_t type, int32 handle, int64 *file_size) if (cm_dbs_get_file_size(handle, file_size) != CT_SUCCESS) { return CT_ERROR; } + } else if (type == DEV_TYPE_RAW) { + if (g_raw_device_op.raw_seek == NULL) { + return CT_ERROR; + } + *file_size = g_raw_device_op.raw_seek(handle, 0, SEEK_END); } else { return CT_ERROR; } @@ -601,15 +612,15 @@ bool32 cm_exist_device_dir(device_type_t type, const char *name) if (type == DEV_TYPE_FILE) { return cm_dir_exist(name); } else if (type == DEV_TYPE_RAW) { - if (g_raw_device_op.raw_exist_dir == NULL) { + if (g_raw_device_op.raw_stat == NULL) { return CT_FALSE; } - bool32 result = CT_FALSE; - if (g_raw_device_op.raw_exist_dir(name, &result) != CT_SUCCESS) { + dss_stat_t item = {0}; + if (g_raw_device_op.raw_stat(name, &item) != CT_SUCCESS) { return CT_FALSE; } - return result; + return CT_TRUE; } else if (type == DEV_TYPE_DBSTOR_FILE) { return cm_dbs_exist_file(name, DIR_TYPE); } else { @@ -675,15 +686,15 @@ bool32 cm_exist_device(device_type_t type, const char *name) if (type == DEV_TYPE_FILE) { return cm_file_exist(name); } else if (type == DEV_TYPE_RAW) { - if (g_raw_device_op.raw_exist == NULL) { + if (g_raw_device_op.raw_stat == NULL) { return CT_FALSE; } - bool32 result = CT_FALSE; - if (g_raw_device_op.raw_exist(name, &result) != CT_SUCCESS) { + dss_stat_t item = {0}; + if (g_raw_device_op.raw_stat(name, &item) != CT_SUCCESS) { return CT_FALSE; } - return result; + return CT_TRUE; } else if (type == DEV_TYPE_ULOG) { return cm_dbs_map_exist(name, DEV_TYPE_ULOG); } else if (type == DEV_TYPE_PGPOOL) { @@ -1062,6 +1073,68 @@ bool32 cm_match_arch_pattern(const char *filename) return CT_FALSE; } +int cm_aio_dss_prep_read(cm_iocb_t *iocb, int fd, void *buf, size_t count, long long offset) +{ + if (SECUREC_UNLIKELY(g_raw_device_op.aio_prep_pread == NULL)) { + CT_LOG_RUN_ERR("File aio_prep_pread function is not defined."); + return CT_ERROR; + } + + if (g_raw_device_op.aio_prep_pread(iocb, fd, buf, count, offset) != CT_SUCCESS) { + CT_THROW_RAW_ERROR; + return CT_ERROR; + } + + return CT_SUCCESS; +} + +int cm_aio_dss_prep_write(cm_iocb_t *iocb, int fd, void *buf, size_t count, long long offset) +{ + if (SECUREC_UNLIKELY(g_raw_device_op.aio_prep_pwrite == NULL)) { + CT_LOG_RUN_ERR("File aio_prep_pread function is not defined."); + return CT_ERROR; + } + + if (g_raw_device_op.aio_prep_pwrite(iocb, fd, buf, count, offset) != CT_SUCCESS) { + CT_THROW_RAW_ERROR; + return CT_ERROR; + } + + return CT_SUCCESS; +} + +int cm_aio_dss_post_write(cm_iocb_t *iocb, int fd, size_t count, long long offset) +{ + if (SECUREC_UNLIKELY(g_raw_device_op.aio_post_pwrite == NULL)) { + CT_LOG_RUN_ERR("File aio_prep_pwrite function is not defined."); + return CT_ERROR; + } + + int ret = g_raw_device_op.aio_post_pwrite(iocb, fd, count, offset); + if (ret != CT_SUCCESS) { + CT_LOG_RUN_ERR("File aio_pre_pwrite execute failed, ret: %d.", ret); + CT_THROW_RAW_ERROR; + return CT_ERROR; + } + + return CT_SUCCESS; +} + +status_t cm_fdatasync_device(device_type_t type, int32 handle) +{ + if (type == DEV_TYPE_FILE) { + return cm_fdatasync_file(handle); + } else if (type == DEV_TYPE_RAW) { + // dss default flag has O_SYNC | O_DIRECT + return CT_SUCCESS; + } else { + CT_LOG_RUN_ERR("File cm_fdatasync_device type %d is not supported.", type); + return CT_ERROR; + } +} + + + #ifdef __cplusplus } #endif diff --git a/pkg/src/common/cm_device.h b/pkg/src/common/cm_device.h index d56a52cf92b976c5646ae67b280ff194f9d93626..060387371863ac255c2fd7470d6dbc43164e1a8c 100644 --- a/pkg/src/common/cm_device.h +++ b/pkg/src/common/cm_device.h @@ -66,7 +66,14 @@ typedef io_callback_t cm_io_callback_t; typedef io_context_t cm_io_context_t; #endif +typedef struct st_iocb_ex_t { + cm_iocb_t obj; // caution:SHOULD be the FIRST + uint64 handle; // the handle of raw interface + uint64 offset; // the offset of raw interface +} cm_iocb_ex_t; + #define CM_IOCB_LENTH (sizeof(cm_iocb_t) + sizeof(cm_iocb_t*) + sizeof(cm_io_event_t)) +#define CM_IOCB_LENTH_EX (sizeof(cm_iocb_ex_t) + sizeof(cm_iocb_ex_t *) + sizeof(cm_io_event_t)) typedef int (*cm_io_setup)(int maxevents, cm_io_context_t *io_ctx); typedef int (*cm_io_destroy)(cm_io_context_t ctx); @@ -90,6 +97,23 @@ typedef struct st_aio_lib { cm_io_getevents io_getevents; }cm_aio_lib_t; +#define DSS_MAX_NAME_LEN 64 /* Consistent with dss_def.h */ +typedef enum en_dss_item_type { DSS_PATH, DSS_FILE, DSS_LINK } dss_item_type_t; +typedef struct st_dss_stat { + unsigned long long size; + unsigned long long written_size; + time_t create_time; + time_t update_time; + char name[DSS_MAX_NAME_LEN]; + dss_item_type_t type; +} dss_stat_t; + +typedef enum en_dss_conn_opt_key { + DSS_CONN_OPT_TIME_OUT = 0, +} dss_conn_opt_key_e; + +typedef struct st_dss_stat *dss_stat_info_t; + #define cm_device_size(type, handle) cm_seek_device((type), (handle), 0, SEEK_END) status_t cm_aio_setup(cm_aio_lib_t *lib_ctx, int maxevents, cm_io_context_t *io_ctx); @@ -101,6 +125,12 @@ void cm_aio_prep_read(cm_iocb_t *iocb, int fd, void *buf, size_t count, long lon void cm_aio_prep_write(cm_iocb_t *iocb, int fd, void *buf, size_t count, long long offset); void cm_aio_set_callback(cm_iocb_t *iocb, cm_io_callback_t cb); +int cm_aio_dss_prep_read(cm_iocb_t *iocb, int fd, void *buf, size_t count, long long offset); +int cm_aio_dss_prep_write(cm_iocb_t *iocb, int fd, void *buf, size_t count, long long offset); +int cm_aio_dss_post_write(cm_iocb_t *iocb, int fd, size_t count, long long offset); + +status_t cm_fdatasync_device(device_type_t type, int32 handle); + device_type_t cm_device_type(const char *name); status_t cm_remove_device(device_type_t type, const char *name); status_t cm_remove_device_when_enoent(device_type_t type, const char *name); @@ -146,13 +176,13 @@ status_t cm_cal_partid_by_pageid(uint64 page_id, uint32 page_size, uint32 *part_ // callback for register raw device typedef status_t (*raw_open_device)(const char *name, uint32 flags, int32 *handle); -typedef status_t (*raw_read_device)(int32 handle, int64 offset, void *buf, int32 size, int32 *read_size); +typedef status_t (*raw_read_device)(int32 handle, void *buf, int32 size, int32 *read_size); typedef status_t (*raw_write_device)(int32 handle, int64 offset, const void *buf, int32 size); typedef int64 (*raw_seek_device)(int32 handle, int64 offset, int32 origin); typedef status_t (*raw_trucate_device)(int32 handle, int64 keep_size); typedef status_t (*raw_create_device)(const char *name, uint32 flags); typedef status_t (*raw_remove_device)(const char *name); -typedef void (*raw_close_device)(int32 *handle); +typedef void (*raw_close_device)(int32 handle); typedef status_t (*raw_exist_device)(const char *name, bool32 *result); typedef status_t (*raw_create_device_dir)(const char *name); typedef status_t (*raw_exist_device_dir)(const char *name, bool32 *result); @@ -160,7 +190,25 @@ typedef status_t (*raw_rename_device)(const char *src, const char *dst); typedef status_t (*raw_check_device_size)(int32 size); typedef status_t (*raw_align_device_size)(int32 size); +typedef void (*raw_device_phy_size)(int32 *handle, int64 *fsize); +typedef void (*raw_error_info)(int32 *errcode, const char **errmsg); +typedef status_t (*raw_pread_device)(int32 handle, void *buf, int32 size, int64 offset, int32 *read_size); +typedef status_t (*raw_pwrite_device)(int32 handle, const void *buf, int32 size, int64 offset); +typedef void (*raw_set_svr_path)(const char *conn_path); +typedef status_t (*raw_aio_prep_pread)(void *iocb, int32 handle, void *buf, size_t count, long long offset); +typedef status_t (*raw_aio_prep_pwrite)(void *iocb, int32 handle, void *buf, size_t count, long long offset); +typedef int (*raw_get_au_size)(int handle, long long *au_size); +typedef void (*device_usr_cb_log_output_t)(int log_type, int log_level, const char *code_file_name, + uint32 code_line_num, const char *module_name, const char *format, ...); +typedef void (*raw_regist_logger)(device_usr_cb_log_output_t log_output, unsigned int log_level); +typedef void (*set_dss_log_level)(unsigned int log_level); +typedef int (*raw_stat)(const char *path, dss_stat_info_t item); +typedef int (*raw_aio_post_pwrite)(void *iocb, int32 handle, size_t count, long long offset); +typedef int (*raw_dss_set_conn_opts)(dss_conn_opt_key_e key, void *value); +typedef int (*raw_dss_set_def_conn_timeout)(int timeout); + typedef struct st_raw_device_op { + void *handle; raw_create_device raw_create; raw_remove_device raw_remove; raw_open_device raw_open; @@ -175,6 +223,20 @@ typedef struct st_raw_device_op { raw_rename_device raw_rename; raw_check_device_size raw_check_size; raw_align_device_size raw_align_size; + raw_device_phy_size raw_fsize_pyhsical; + raw_error_info raw_get_error; + raw_pread_device raw_pread; + raw_pwrite_device raw_pwrite; + raw_set_svr_path raw_set_svr_path; + raw_regist_logger raw_regist_logger; + raw_aio_prep_pread aio_prep_pread; + raw_aio_prep_pwrite aio_prep_pwrite; + raw_get_au_size get_au_size; + raw_stat raw_stat; + set_dss_log_level set_dss_log_level; + raw_aio_post_pwrite aio_post_pwrite; + raw_dss_set_conn_opts dss_set_conn_opts; + raw_dss_set_def_conn_timeout dss_set_def_conn_timeout; } raw_device_op_t; // interface for register raw device callback function diff --git a/pkg/src/common/cm_dlock.c b/pkg/src/common/cm_dlock.c index a1abfd5d34e316bfd9dc5cb3f9342d7e855ffd60..b2a3f1b09daae99855ac85f88ec5a9a644eefc1b 100644 --- a/pkg/src/common/cm_dlock.c +++ b/pkg/src/common/cm_dlock.c @@ -533,7 +533,7 @@ status_t cm_disk_unlock(dlock_t *lock, int32 fd) #else status_t status = CT_SUCCESS; - status = cm_disk_unlock_interal(lock, fd, CT_TRUE); + status = cm_disk_unlock_interal(lock, fd, CT_FALSE); if (CT_SUCCESS != status) { return status; } diff --git a/pkg/src/common/cm_error.c b/pkg/src/common/cm_error.c index 59d872ec54d2d3078d53874c4a7da0505a65a340..182e08bc91d52987e3de2aa670729a8417df9538 100644 --- a/pkg/src/common/cm_error.c +++ b/pkg/src/common/cm_error.c @@ -1092,6 +1092,7 @@ const char *g_error_desc[] = { [ERR_CMS_CONVERT_VALUE] = "cannot convert '%s' to %s type", [ERR_ACCESS_DEPOSIT_INST] = "failed to access deposit instance, logical id %d physical id %d", + [ERR_DSS_FAILED] = "failed when calling dss interface, error code: %d, error message = %s", [ERR_CLUSTER_DDL_DISABLED] = "cluster DDL operation not available, please check whether other instances are open", [ERR_CM_OPEN_DISK] = "open disk failed, errno %d", @@ -1461,6 +1462,7 @@ status_t cm_set_sql_error(const char *file, uint32 line, ct_errno_t code, const { char log_msg[CT_MAX_LOG_CONTENT_LENGTH] = { 0 }; char *last_file = NULL; + char *file_name = NULL; log_param_t *log_param = cm_log_param_instance(); errno_t err; #ifdef WIN32 @@ -1468,16 +1470,21 @@ status_t cm_set_sql_error(const char *file, uint32 line, ct_errno_t code, const #else last_file = strrchr(file, '/'); #endif + if (last_file == NULL) { + file_name = (char *)file; + } else { + file_name = last_file + 1; + } err = vsnprintf_s(log_msg, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH - 1, format, args); if (SECUREC_UNLIKELY(err == -1)) { CT_LOG_RUN_ERR("Secure C lib has thrown an error %d while setting error", err); } if (!g_tls_error.is_ignore_log) { - CT_LOG_DEBUG_ERR("CT-%05d : %s [%s:%u]", code, log_msg, last_file + 1, line); + CT_LOG_DEBUG_ERR("CT-%05d : %s [%s:%u]", code, log_msg, file_name, line); if (log_param->log_instance_startup || code == ERR_SYSTEM_CALL) { - CT_LOG_RUN_ERR("CT-%05d : %s [%s:%u]", code, log_msg, last_file + 1, line); + CT_LOG_RUN_ERR("CT-%05d : %s [%s:%u]", code, log_msg, file_name, line); } } diff --git a/pkg/src/common/cm_error.h b/pkg/src/common/cm_error.h index 0f76b1fca651ccfcc5f1d2fde0eb2b2d534e20f8..967136dc30f0523dbaced751be096af26ca07833 100644 --- a/pkg/src/common/cm_error.h +++ b/pkg/src/common/cm_error.h @@ -1106,6 +1106,7 @@ typedef enum en_errno { ERR_CMS_VOTEDISK_PATH = 2825, ERR_ACCESS_DEPOSIT_INST = 2900, + ERR_DSS_FAILED = 2901, /* DCS */ ERR_DCS_REQ_PAGE_FAILED = 3000, diff --git a/pkg/src/common/cm_file.c b/pkg/src/common/cm_file.c index 56fa3b54baccef3aaf037a25dc21d055f2dd1836..3b857ae50953687a0d3ab5c548703213db836b42 100644 --- a/pkg/src/common/cm_file.c +++ b/pkg/src/common/cm_file.c @@ -75,24 +75,24 @@ static char *cm_skip_drive(const char *path) status_t cm_fsync_file(int32 file) { -#ifndef WIN32 - if (fsync(file) != 0) { - CT_THROW_ERROR(ERR_DATAFILE_FSYNC, errno); - return CT_ERROR; - } -#endif +//#ifndef WIN32 +// if (fsync(file) != 0) { +// CT_THROW_ERROR(ERR_DATAFILE_FSYNC, errno); +// return CT_ERROR; +// } +//#endif return CT_SUCCESS; } status_t cm_fdatasync_file(int32 file) { -#ifndef WIN32 - if (fdatasync(file) != 0) { - CT_THROW_ERROR(ERR_DATAFILE_FDATASYNC, errno); - return CT_ERROR; - } -#endif +//#ifndef WIN32 +// if (fdatasync(file) != 0) { +// CT_THROW_ERROR(ERR_DATAFILE_FDATASYNC, errno); +// return CT_ERROR; +// } +//#endif return CT_SUCCESS; } diff --git a/pkg/src/common/cm_file_iofence.c b/pkg/src/common/cm_file_iofence.c index 1529ca10be4150e51d2af467597b28ba71d74d92..3fd5ef8eb60843189ada6b57642960a81bb33f54 100644 --- a/pkg/src/common/cm_file_iofence.c +++ b/pkg/src/common/cm_file_iofence.c @@ -72,6 +72,7 @@ void iof_file_watcher(thread_t *thread) status_t cm_file_iof_register(uint32 inst_id, thread_t *file_iof_thd) { + return CT_SUCCESS; if (strlen(g_file_cfg.file_path) == 0) { CT_LOG_RUN_WAR("Iof file path is empty, no need iofence."); return CT_SUCCESS; diff --git a/pkg/src/common/cm_log.c b/pkg/src/common/cm_log.c index 17969ab3f14db644c1d2219f763a3653dc7d1b37..d142e3e20810263f5246481f0405f7fab76edd83 100644 --- a/pkg/src/common/cm_log.c +++ b/pkg/src/common/cm_log.c @@ -957,7 +957,53 @@ void cm_write_normal_log(log_id_t log_id, log_level_t log_level, const char *cod } cm_log_build_normal_head((char *)buf, sizeof(buf), log_level, g_module_info[module_id]); - + va_list args; + va_start(args, format); + buf_text.str = buf; + buf_text.len = (uint32)strlen(buf); + if (errcode >= 0) { + cm_log_fulfil_write_buf(log_file_handle, &buf_text, sizeof(buf), need_rec_filelog, new_format, args); + } else { + // if the security function fails, continue to write the log after the string is truncated + cm_log_fulfil_write_buf(log_file_handle, &buf_text, sizeof(buf), need_rec_filelog, new_format, args); + } + va_end(args); +} + +void cm_write_normal_log2(log_id_t log_id, log_level_t log_level, const char *code_file_name, uint32 code_line_num, + const int module_id, bool32 need_rec_filelog, const char *format, ...) +{ + int32 error_code = 0; + char buf[CT_MAX_LOG_CONTENT_LENGTH + CT_MAX_LOG_HEAD_LENGTH + 2] = { 0 }; + char new_format[CT_MAX_LOG_CONTENT_LENGTH] = { 0 }; + log_file_handle_t *log_file_handle = &g_logger[log_id]; + text_t buf_text; + //char *last_file = NULL; + const char *err_msg = NULL; + log_param_t *log_param = cm_log_param_instance(); + errno_t errcode; + + if (log_param->log_instance_startup) { + errcode = snprintf_s(new_format, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH - 1, "%s [%s:%u]", format, + code_file_name, code_line_num); + } else { + if (log_id == LOG_RUN) { + cm_get_error(&error_code, &err_msg, NULL); + } + + if (error_code == 0 || need_rec_filelog == CT_FALSE) { + errcode = snprintf_s(new_format, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH - 1, "%s [%s:%u]", + format, code_file_name, code_line_num); + } else if (error_code == ERR_ASSERT_ERROR) { + errcode = snprintf_s(new_format, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH - 1, + "CT-%05d:%s,%s [%s:%u]", error_code, format, err_msg, code_file_name, code_line_num); + } else { + errcode = snprintf_s(new_format, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH - 1, + "CT-%05d:%s,%s [%s:%u]", error_code, format, err_msg, code_file_name, code_line_num); + } + } + + cm_log_build_normal_head((char *)buf, sizeof(buf), log_level, g_module_info[module_id]); va_list args; va_start(args, format); buf_text.str = buf; diff --git a/pkg/src/common/cm_log.h b/pkg/src/common/cm_log.h index a3b5552c503c09c4fa43ffc0a422de6de59b0e1d..85a4ed81d5577a26b3d5367c0a810723caa41ecb 100644 --- a/pkg/src/common/cm_log.h +++ b/pkg/src/common/cm_log.h @@ -78,7 +78,7 @@ typedef enum en_module_id { KNL_COMM, CMS, MES, - ZENFS, + DSSAPI, EXT_PROC, BACKUP, ARCHIVE, @@ -202,6 +202,8 @@ void cm_write_blackbox_log(const char *format, ...) CT_CHECK_FMT(1, 2); void cm_write_normal_log(log_id_t log_id, log_level_t log_level, const char *code_file_name, uint32 code_line_num, const int module_id, bool32 need_rec_filelog, const char *format, ...) CT_CHECK_FMT(7, 8); +void cm_write_normal_log2(log_id_t log_id, log_level_t log_level, const char *code_file_name, uint32 code_line_num, + const int module_id, bool32 need_rec_filelog, const char *format, ...) CT_CHECK_FMT(7, 8); void cm_write_oper_log(char *buf, uint32 len); void cm_write_trace_log(const char *format, ...); void cm_fync_logfile(void); diff --git a/pkg/src/common/cm_scsi.c b/pkg/src/common/cm_scsi.c index a433329d58dfc25885c41eb5aef8baa0d580ba88..d7bbf6d12ac88869f71920697f1815bf53a5a6ae 100644 --- a/pkg/src/common/cm_scsi.c +++ b/pkg/src/common/cm_scsi.c @@ -586,7 +586,7 @@ int32 cm_scsi3_caw(int32 fd, int64 block_addr, char *buff, int32 buff_len) status = ioctl(fd, SG_IO, &hdr); if (status < 0) { - CT_LOG_DEBUG_ERR("Sending SCSI caw command failed, status %d, errno %d.", status, errno); + CT_LOG_RUN_ERR("Sending SCSI caw command failed, status %d, errno %d.", status, errno); return CT_ERROR; } diff --git a/pkg/src/common/cm_system.c b/pkg/src/common/cm_system.c index b3b3981837a101434650a68a3687ff36d430f44e..3ca42e3ea801aca7253ace2e6003e33a99454832 100644 --- a/pkg/src/common/cm_system.c +++ b/pkg/src/common/cm_system.c @@ -265,7 +265,7 @@ static time_t cm_convert_filetime(FILETIME *ft) } #endif -int64 cm_sys_process_start_time(uint64 pid) +int64 cm_sys_process_start_time_s(uint64 pid) { #ifdef WIN32 FILETIME create_time, exit_time, kernel_time, user_time; @@ -332,7 +332,7 @@ int64 cm_sys_process_start_time(uint64 pid) bool32 cm_sys_process_alived(uint64 pid, int64 start_time) { - int64 process_time = cm_sys_process_start_time(pid); + int64 process_time = cm_sys_process_start_time_s(pid); #ifdef WIN32 return (llabs(start_time - process_time) <= 1); diff --git a/pkg/src/common/cm_system.h b/pkg/src/common/cm_system.h index 95ce2b934ac8d79cc4b1ded2cd576bc26c7d937b..23c6ba656551fa4f2da54ed610f5fe11b96b2fdb 100644 --- a/pkg/src/common/cm_system.h +++ b/pkg/src/common/cm_system.h @@ -41,7 +41,7 @@ char *cm_sys_user_name(void); char *cm_sys_host_name(void); char *cm_sys_platform_name(void); int64 cm_sys_ticks(void); -int64 cm_sys_process_start_time(uint64 pid); +int64 cm_sys_process_start_time_s(uint64 pid); bool32 cm_sys_process_alived(uint64 pid, int64 start_time); void cm_try_init_system(void); uint32 cm_sys_get_nprocs(void); diff --git a/pkg/src/kernel/buffer/knl_buffer.c b/pkg/src/kernel/buffer/knl_buffer.c index 1eecf2beed914ec002c656f3b55cddd72cf2fb79..5390c69f9b7ad5ebcff529211dcb6aefa59101f1 100644 --- a/pkg/src/kernel/buffer/knl_buffer.c +++ b/pkg/src/kernel/buffer/knl_buffer.c @@ -84,7 +84,7 @@ status_t buf_init(knl_session_t *session) } if (kernel->attr.enable_asynch) { - return buf_aio_init(session); + // return buf_aio_init(session); } cm_init_thread_lock(&ctx->buf_mutex); diff --git a/pkg/src/kernel/buffer/knl_buffer_access.c b/pkg/src/kernel/buffer/knl_buffer_access.c index e8b272cebb4199adf4713dd8a677f20edd1fa5f5..c0ebbaa49623fd075ddd2535eb6b8683b437c7bc 100644 --- a/pkg/src/kernel/buffer/knl_buffer_access.c +++ b/pkg/src/kernel/buffer/knl_buffer_access.c @@ -893,7 +893,7 @@ status_t buf_read_page_asynch(knl_session_t *session, page_id_t page_id) buf_ctrl_t *ctrl = NULL; page_id_t head; bool32 is_compress = page_compress(session, page_id); - + return CT_SUCCESS; if (!session->kernel->attr.enable_asynch) { return CT_SUCCESS; } @@ -1741,10 +1741,10 @@ status_t buf_read_prefetch_page(knl_session_t *session, page_id_t page_id, latch buf_log_enter_page(session, ctrl, mode, options); if (session->kernel->attr.enable_asynch) { - if (buf_try_prefetch_next_ext(session, ctrl) != CT_SUCCESS) { + /* if (buf_try_prefetch_next_ext(session, ctrl) != CT_SUCCESS) { CT_LOG_DEBUG_WAR("[BUFFER] failed to prefetch next extent file : %u , page: %llu", (uint32)ctrl->page_id.file, (uint64)ctrl->page_id.page); - } + }*/ } return CT_SUCCESS; diff --git a/pkg/src/kernel/common/knl_context.c b/pkg/src/kernel/common/knl_context.c index 9b160786f189660e63d9050454fe996780fe5464..6d559a9f08e683bf751a326b779bcf477ec72380 100644 --- a/pkg/src/kernel/common/knl_context.c +++ b/pkg/src/kernel/common/knl_context.c @@ -89,11 +89,12 @@ status_t knl_startup(knl_handle_t kernel) } if (DB_ATTR_CLUSTER(session)) { - if (g_knl_callback.device_init((const char *)session->kernel->dtc_attr.ctstore_inst_path) != CT_SUCCESS) { + cm_dbs_cfg_s *cfg = cm_dbs_get_cfg(); + if (cfg->enable && + g_knl_callback.device_init((const char *)session->kernel->dtc_attr.ctstore_inst_path) != CT_SUCCESS) { CT_LOG_RUN_INF("RAFT: db init raw type device failed"); return CT_ERROR; } - cm_dbs_cfg_s *cfg = cm_dbs_get_cfg(); if (!cfg->enable) { CT_LOG_RUN_INF("Note: dbstore is not enabled, the disaster recovery funcs would not work."); } else { diff --git a/pkg/src/kernel/knl_database.c b/pkg/src/kernel/knl_database.c index 638d460c7def64565ebb8a771884ea6fc3c76a82..76087baa0f78576e90df5a2ef64565ddec963082 100644 --- a/pkg/src/kernel/knl_database.c +++ b/pkg/src/kernel/knl_database.c @@ -218,6 +218,7 @@ status_t db_load_logfiles(knl_session_t *session) SYNC_POINT_GLOBAL_START(CANTIAN_OPEN_DEVICE_FAIL, &ret, CT_ERROR); /* logfile can be opened for a long time, closed in db_close_log_files */ ret = cm_open_device(logfile->ctrl->name, logfile->ctrl->type, knl_redo_io_flag(session), &logfile->handle); + CT_LOG_RUN_INF_LIMIT(60, "[ARCH_TEST] adr %p, name %s, type %u, mode %u", logfile, logfile->ctrl->name, logfile->ctrl->type, knl_redo_io_flag(session)); SYNC_POINT_GLOBAL_END; if (ret != CT_SUCCESS) { CT_LOG_RUN_ERR("[DB] failed to open %s ", logfile->ctrl->name); @@ -503,7 +504,7 @@ void db_close(knl_session_t *session, bool32 need_ckpt) } if (session->kernel->attr.enable_asynch) { - cm_close_thread(&session->kernel->buf_aio_ctx.thread); + // cm_close_thread(&session->kernel->buf_aio_ctx.thread); } cm_close_timer(g_timer()); @@ -776,9 +777,9 @@ static status_t db_start_daemon(knl_session_t *session) } if (kernel->attr.enable_asynch) { - if (cm_create_thread(buf_aio_proc, 0, kernel, &kernel->buf_aio_ctx.thread) != CT_SUCCESS) { + /* if (cm_create_thread(buf_aio_proc, 0, kernel, &kernel->buf_aio_ctx.thread) != CT_SUCCESS) { return CT_ERROR; - } + }*/ } if (cm_create_thread(rmon_proc, 0, kernel->sessions[SESSION_ID_RMON], &kernel->rmon_ctx.thread) != CT_SUCCESS) { diff --git a/pkg/src/kernel/persist/knl_archive.c b/pkg/src/kernel/persist/knl_archive.c index e2793d2a68e37a88acfaddeaea43ee31b95f190e..ca3d13e3c2c074ce5580be8b5cdb324548fa0e38 100644 --- a/pkg/src/kernel/persist/knl_archive.c +++ b/pkg/src/kernel/persist/knl_archive.c @@ -1123,7 +1123,7 @@ status_t arch_dbstor_rename_tmp_file(const char *tmp_file_name, const char *arch void arch_set_force_archive_stat(arch_proc_context_t *proc_ctx, bool32 result) { - CT_LOG_RUN_INF("[ARCH] force archive failed, proc data type is %u", proc_ctx->data_type); + CT_LOG_RUN_INF_LIMIT(60, "[ARCH] force archive failed, proc data type is %u", proc_ctx->data_type); if (proc_ctx->data_type == ARCH_DATA_TYPE_DBSTOR_STANDBY) { proc_ctx->force_archive_failed = result; return; @@ -1479,9 +1479,10 @@ static status_t arch_archive_tmp_file(knl_session_t *session, aligned_buf_t buf, status_t status = CT_SUCCESS; SYNC_POINT_GLOBAL_START(CANTIAN_REFORM_ARCHIVE_READ_REDO_LOG_FAIL, &status, CT_ERROR); status = cm_open_device(logfile->ctrl->name, logfile->ctrl->type, knl_redo_io_flag(session), &arch_files.src_file); + CT_LOG_RUN_INF_LIMIT(60, "[ARCH_TEST] adr %p, name %s, type %u, mode %u", logfile, logfile->ctrl->name, logfile->ctrl->type, knl_redo_io_flag(session)); SYNC_POINT_GLOBAL_END; if (status != CT_SUCCESS) { - CT_LOG_RUN_ERR("[ARCH] failed to open log file %s", logfile->ctrl->name); + CT_LOG_RUN_ERR_LIMIT(60, "[ARCH] failed to open log file %s", logfile->ctrl->name); return CT_ERROR; } @@ -1902,6 +1903,7 @@ bool32 arch_check_logfile(knl_session_t *session, arch_proc_context_t *proc_ctx, void arch_file_archive(knl_session_t *session, arch_proc_context_t *proc_ctx) { log_file_t *logfile = session->kernel->redo_ctx.files + proc_ctx->next_file_id; + CT_LOG_RUN_INF_LIMIT(60, "[ARCH_TEST] adr %p, id %u", logfile, proc_ctx->next_file_id); char arch_file_name[CT_FILE_NAME_BUFFER_SIZE] = {0}; if (arch_check_logfile(session, proc_ctx, logfile) == CT_TRUE) { return; @@ -3266,7 +3268,7 @@ status_t arch_check_dest(arch_context_t *arch_ctx, char *dest, uint32 cur_pos) if ((attr->arch_attr[cur_pos].dest_mode == LOG_ARCH_DEST_LOCATION) && !cm_exist_device_dir(arch_get_device_type(dest), dest)) { - if (cm_dbs_get_deploy_mode() == DBSTOR_DEPLOY_MODE_NAS) { + if (cm_dbs_is_enable_dbs() && cm_dbs_get_deploy_mode() == DBSTOR_DEPLOY_MODE_NAS) { CT_THROW_ERROR(ERR_DIR_NOT_EXISTS, dest); return CT_ERROR; } diff --git a/pkg/src/kernel/persist/knl_ckpt.c b/pkg/src/kernel/persist/knl_ckpt.c index cc618e94bff7a310fd26ff7fb31049068e3b2f15..33c290f7330d90eba77a6f1772e512db56202426 100644 --- a/pkg/src/kernel/persist/knl_ckpt.c +++ b/pkg/src/kernel/persist/knl_ckpt.c @@ -150,7 +150,12 @@ status_t ckpt_init(knl_session_t *session) } if (kernel->attr.enable_asynch) { - ctx->group.iocbs_buf = (char *)malloc(CT_CKPT_GROUP_SIZE(session) * CM_IOCB_LENTH); + if (DB_IS_CLUSTER(session)) { + ctx->group.iocbs_buf = (char *)malloc(CT_CKPT_GROUP_SIZE(session) * CM_IOCB_LENTH_EX); + } else { + ctx->group.iocbs_buf = (char *)malloc(CT_CKPT_GROUP_SIZE(session) * CM_IOCB_LENTH); + } + if (ctx->group.iocbs_buf == NULL) { CT_LOG_RUN_ERR("[CKPT] iocb malloc fail"); return CT_ERROR; @@ -1546,6 +1551,7 @@ static status_t dbwr_async_io_write(knl_session_t *session, cm_aio_iocbs_t *aio_ uint32 idx = 0; errno_t ret; datafile_t *df = NULL; + uint32 write_size; ret = memset_sp(dbwr->flags, sizeof(dbwr->flags), 0, sizeof(dbwr->flags)); knl_securec_check(ret); @@ -1562,9 +1568,20 @@ static status_t dbwr_async_io_write(knl_session_t *session, cm_aio_iocbs_t *aio_ page = (page_head_t *)(ctx->group.buf + ((uint64)buf_id) * size); knl_panic(item->ctrl != NULL); knl_panic(IS_SAME_PAGID(item->ctrl->page_id, AS_PAGID(page->id))); - aio_cbs->iocb_ptrs[cb_id] = &aio_cbs->iocbs[cb_id]; - cm_aio_prep_write(aio_cbs->iocb_ptrs[cb_id], *asyncio_ctx->handles[idx], (void *)page, size, - (int64)asyncio_ctx->offsets[idx]); + if (asyncio_ctx->datafiles[idx]->ctrl->type == DEV_TYPE_RAW) { + cm_iocb_ex_t *iocb_ex = (cm_iocb_ex_t *)aio_cbs->iocbs; + iocb_ex[cb_id].handle = *asyncio_ctx->handles[idx]; + iocb_ex[cb_id].offset = (int64)asyncio_ctx->offsets[idx]; + aio_cbs->iocb_ptrs[cb_id] = &iocb_ex[cb_id].obj; + int ret = cm_aio_dss_prep_write(aio_cbs->iocb_ptrs[cb_id], *asyncio_ctx->handles[idx], (void *)page, + size, (int64)asyncio_ctx->offsets[idx]); + knl_panic_log(ret == 0, "[CKPT] ABORT INFO:failed to write page by dss aio"); + CT_LOG_DEBUG_INF("[CKPT] prepare write datafile %u handle is %llu.", cb_id, iocb_ex[cb_id].handle); + } else { + aio_cbs->iocb_ptrs[cb_id] = &aio_cbs->iocbs[cb_id]; + cm_aio_prep_write(aio_cbs->iocb_ptrs[cb_id], *asyncio_ctx->handles[idx], (void *)page, size, + (int64)asyncio_ctx->offsets[idx]); + } knl_panic(asyncio_ctx->offsets[idx] == (uint64)item->ctrl->page_id.page * PAGE_SIZE(*page)); cb_id++; @@ -1592,10 +1609,16 @@ static status_t dbwr_async_io_write(knl_session_t *session, cm_aio_iocbs_t *aio_ return CT_ERROR; } for (int32 i = 0; i < aio_ret; i++) { - if (aio_cbs->events[i].res != size) { - CT_LOG_RUN_ERR("[CKPT] failed to write by event, error code: %ld", aio_cbs->events[i].res); + write_size = aio_cbs->events[i].obj->u.c.nbytes; + if (aio_cbs->events[i].res != write_size) { + CT_LOG_RUN_ERR("[CKPT] failed to write by event, res: %ld, size: %u", aio_cbs->events[i].res, write_size); return CT_ERROR; } + cm_iocb_ex_t *iocb_ex = (cm_iocb_ex_t *)aio_cbs->events[i].obj; + CT_LOG_DEBUG_INF("[CKPT] post write datafile %u, aio_ret: %u, handle: %llu, offset: %llu.", + i, aio_ret, iocb_ex->handle, iocb_ex->offset); + ret = cm_aio_dss_post_write(&iocb_ex->obj, iocb_ex->handle, aio_cbs->events[i].obj->u.c.nbytes, iocb_ex->offset); + knl_panic_log(ret == 0, "[CKPT] failed to post write by async io, error code: %d, aio_idx: %d", ret, i); } event_num = event_num - aio_ret; } @@ -1647,13 +1670,21 @@ static status_t dbwr_flush_async_io(knl_session_t *session, dbwr_context_t *dbwr CT_LOG_RUN_ERR("[CKPT] failed to open datafile %s", asyncio_ctx->datafiles[latch_cnt]->ctrl->name); return CT_ERROR; } + CT_LOG_DEBUG_INF("[CKPT] datafile %u handle is %u.", latch_cnt, *asyncio_ctx->handles[latch_cnt]); latch_cnt++; } - buf_offset = dbwr->begin * CM_IOCB_LENTH; - aio_cbs.iocbs = (cm_iocb_t *)(ctx->group.iocbs_buf + buf_offset); - buf_offset += sizeof(cm_iocb_t) * dbwr->io_cnt; - aio_cbs.events = (cm_io_event_t*)(ctx->group.iocbs_buf + buf_offset); + if (DB_IS_CLUSTER(session)) { + buf_offset = dbwr->begin * CM_IOCB_LENTH_EX; + aio_cbs.iocbs = (cm_iocb_t *)(ctx->group.iocbs_buf + buf_offset); + buf_offset += sizeof(cm_iocb_ex_t) * dbwr->io_cnt; + } else { + buf_offset = dbwr->begin * CM_IOCB_LENTH; + aio_cbs.iocbs = (cm_iocb_t *)(ctx->group.iocbs_buf + buf_offset); + buf_offset += sizeof(cm_iocb_t) * dbwr->io_cnt; + } + + aio_cbs.events = (cm_io_event_t *)(ctx->group.iocbs_buf + buf_offset); buf_offset += sizeof(cm_io_event_t) * dbwr->io_cnt; aio_cbs.iocb_ptrs = (cm_iocb_t**)(ctx->group.iocbs_buf + buf_offset); @@ -2318,7 +2349,7 @@ status_t dbwr_fdatasync(knl_session_t *session, dbwr_context_t *dbwr) for (uint32 i = 0; i < CT_MAX_DATA_FILES; i++) { if (dbwr->flags[i]) { - if (cm_fdatasync_file(dbwr->datafiles[i]) != CT_SUCCESS) { + if (cm_fdatasync_device(db->datafiles[i].ctrl->type, dbwr->datafiles[i]) != CT_SUCCESS) { CT_LOG_RUN_ERR("failed to fdatasync datafile %s", db->datafiles[i].ctrl->name); return CT_ERROR; } diff --git a/pkg/src/kernel/persist/knl_log_file.c b/pkg/src/kernel/persist/knl_log_file.c index d442cb3147c679996f20c97a2cf952fe01dc08ea..fdede60b9af455a3bd2d2a985cbc1031b8b39d50 100644 --- a/pkg/src/kernel/persist/knl_log_file.c +++ b/pkg/src/kernel/persist/knl_log_file.c @@ -217,7 +217,7 @@ status_t db_alter_add_logfile(knl_session_t *session, knl_alterdb_def_t *def) cm_remove_device(logfile->ctrl->type, logfile->ctrl->name); return CT_ERROR; } - + CT_LOG_RUN_INF_LIMIT(60, "[ARCH_TEST] adr %p, name %s, type %u, mode %u", logfile, logfile->ctrl->name, logfile->ctrl->type, knl_redo_io_flag(session)); if (lsnd_open_specified_logfile(session, slot) != CT_SUCCESS) { cm_close_device(logfile->ctrl->type, &logfile->handle); cm_remove_device(logfile->ctrl->type, logfile->ctrl->name); diff --git a/pkg/src/protocol/cs_ipc.c b/pkg/src/protocol/cs_ipc.c index d34ba0ee5f6a8a103e98553abb44b5217bb4c492..04f82972892738476e1acaea1fccb556293091d8 100644 --- a/pkg/src/protocol/cs_ipc.c +++ b/pkg/src/protocol/cs_ipc.c @@ -338,7 +338,7 @@ status_t cs_init_ipc_context(const char *ipc_name) return CT_ERROR; } - g_ipc_context.start_time = cm_sys_process_start_time(req.client_info.pid); + g_ipc_context.start_time = cm_sys_process_start_time_s(req.client_info.pid); code = cs_request_ipc_lsnr(&req, &token); if (code != CT_SUCCESS) { diff --git a/pkg/src/server/params/load_server.c b/pkg/src/server/params/load_server.c index 43f8dac0bb3f23fbae16201d665b140a59cedb51..8ae428dacfe70b935bbdf124a4f0e5579890c794 100644 --- a/pkg/src/server/params/load_server.c +++ b/pkg/src/server/params/load_server.c @@ -37,6 +37,7 @@ #include "cm_log.h" #include "cm_file_iofence.h" #include "kmc_init.h" +#include "srv_device_adpt.h" extern bool32 g_enable_fdsa; extern bool32 g_crc_verify; @@ -651,6 +652,28 @@ static status_t srv_get_mq_cfg(mq_cfg_s *cfg) return CT_SUCCESS; } +status_t srv_load_dss_path() +{ + char *value = NULL; + MEMS_RETURN_IFERR(memset_s(g_instance->kernel.dtc_attr.ctstore_inst_path, CT_UNIX_PATH_MAX, 0, CT_UNIX_PATH_MAX)); + value = srv_get_param("CTSTORE_INST_PATH"); + if (value == NULL) { + cm_reset_error(); + CT_THROW_ERROR(ERR_INVALID_PARAMETER, "CTSTORE_INST_PATH"); + return CT_ERROR; + } + uint32 val_len = (uint32)strlen(value); + if (val_len >= CT_UNIX_PATH_MAX || val_len == 0) { + CT_THROW_ERROR(ERR_INVALID_PARAMETER, "CTSTORE_INST_PATH"); + return CT_ERROR; + } else { + CT_PRINT_IFERR2(snprintf_s(g_instance->kernel.dtc_attr.ctstore_inst_path, CT_UNIX_PATH_MAX, + CT_UNIX_PATH_MAX - 1, "%s", value), + "CTSTORE_INST_PATH", CT_UNIX_PATH_MAX - 1); + } + return CT_SUCCESS; +} + status_t srv_load_server_params(void) { session_pool_t *session_pool = NULL; @@ -980,6 +1003,13 @@ status_t srv_load_server_params(void) CT_RETURN_IFERR(srv_load_keyfiles()); } + g_instance->attr.enable_dbstor = enable_dbstor; + + if (!enable_dbstor) { + CT_RETURN_IFERR(srv_load_dss_path()); + CT_RETURN_IFERR(srv_device_init(g_instance->kernel.dtc_attr.ctstore_inst_path)); + } + CT_RETURN_IFERR(srv_get_param_bool32("ENABLE_LOCAL_INFILE", &g_instance->attr.enable_local_infile)); CT_RETURN_IFERR(srv_get_param_bool32("_STRICT_CASE_DATATYPE", &g_instance->sql.strict_case_datatype)); diff --git a/pkg/src/server/srv_device_adpt.c b/pkg/src/server/srv_device_adpt.c new file mode 100644 index 0000000000000000000000000000000000000000..befcac00339fe0af8ab577450cbd8f38f48f5acb --- /dev/null +++ b/pkg/src/server/srv_device_adpt.c @@ -0,0 +1,97 @@ +/* ------------------------------------------------------------------------- + * This file is part of the Cantian project. + * Copyright (c) 2024 Huawei Technologies Co.,Ltd. + * + * Cantian is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * srv_device_adpt.c + * + * + * IDENTIFICATION + * src/server/srv_device_adpt.c + * + * ------------------------------------------------------------------------- + */ + +#include "srv_device_adpt.h" +#include "cm_device.h" +#include "cm_log.h" +#include "cm_utils.h" + +#ifdef WIN32 +#define DSSAPI "dssapi.dll" +#else +#define DSSAPISO "libdssapi.so" +#endif + +static void srv_dss_write_normal_log(int log_id, int log_level, const char *code_file_name, uint32 code_line_num, + const char *module_name, const char *format, ...) +{ + log_id_t db_log_id = (log_id_t)log_id; + log_level_t db_log_level = (log_level_t)log_level; + va_list args; + va_start(args, format); + char buf[CT_MAX_LOG_CONTENT_LENGTH]; + int32 errcode = vsnprintf_s(buf, CT_MAX_LOG_CONTENT_LENGTH, CT_MAX_LOG_CONTENT_LENGTH, format, args); + if (errcode < 0) { + va_end(args); + return; + } + va_end(args); + + cm_write_normal_log2(db_log_id, db_log_level, code_file_name, code_line_num, DSSAPI, CT_TRUE, buf); +} + +status_t srv_device_init(const char *conn_path) +{ + raw_device_op_t device_op = { 0 }; + status_t ret = cm_open_dl(&device_op.handle, DSSAPISO); + if (ret != CT_SUCCESS) { + return ret; + } + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fcreate", (void **)&device_op.raw_create)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fclose", (void **)&device_op.raw_close)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fread", (void **)&device_op.raw_read)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fopen", (void **)&device_op.raw_open)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fremove", (void **)&device_op.raw_remove)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fseek", (void **)&device_op.raw_seek)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fwrite", (void **)&device_op.raw_write)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_dmake", (void **)&device_op.raw_create_dir)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_frename", (void **)&device_op.raw_rename)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_check_size", (void **)&device_op.raw_check_size)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_align_size", (void **)&device_op.raw_align_size)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_fsize_physical", (void **)&device_op.raw_fsize_pyhsical)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_get_error", (void **)&device_op.raw_get_error)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_pread", (void **)&device_op.raw_pread)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_pwrite", (void **)&device_op.raw_pwrite)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_ftruncate", (void **)&device_op.raw_truncate)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_set_svr_path", (void **)&device_op.raw_set_svr_path)); + CT_RETURN_IFERR( + cm_load_symbol(device_op.handle, "dss_register_log_callback", (void **)&device_op.raw_regist_logger)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_aio_prep_pread", (void **)&device_op.aio_prep_pread)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_aio_prep_pwrite", (void **)&device_op.aio_prep_pwrite)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_get_au_size", (void **)&device_op.get_au_size)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_stat", (void **)&device_op.raw_stat)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_set_log_level", (void **)&device_op.set_dss_log_level)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_aio_post_pwrite", (void **)&device_op.aio_post_pwrite)); + CT_RETURN_IFERR(cm_load_symbol(device_op.handle, "dss_set_conn_opts", (void **)&device_op.dss_set_conn_opts)); + CT_RETURN_IFERR( + cm_load_symbol(device_op.handle, "dss_set_default_conn_timeout", (void **)&device_op.dss_set_def_conn_timeout)); + if (device_op.handle != NULL) { + cm_raw_device_register(&device_op); + device_op.raw_set_svr_path(conn_path); + device_op.raw_regist_logger(srv_dss_write_normal_log, cm_log_param_instance()->log_level); + } + + return CT_SUCCESS; +} \ No newline at end of file diff --git a/pkg/src/server/srv_device_adpt.h b/pkg/src/server/srv_device_adpt.h new file mode 100644 index 0000000000000000000000000000000000000000..62e7512c75ff66f2d3158018d0944b78307b1576 --- /dev/null +++ b/pkg/src/server/srv_device_adpt.h @@ -0,0 +1,46 @@ +/* ------------------------------------------------------------------------- + * This file is part of the Cantian project. + * Copyright (c) 2024 Huawei Technologies Co.,Ltd. + * + * Cantian is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * srv_device_adpt.h + * + * + * IDENTIFICATION + * src/server/srv_device_adpt.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __SS_DEVICE_ADPT_H__ +#define __SS_DEVICE_ADPT_H__ + +#include "cm_defs.h" + +typedef enum en_dss_log_level { + DSS_LOG_LEVEL_ERROR = 0, // error conditions + DSS_LOG_LEVEL_WARN, // warning conditions + DSS_LOG_LEVEL_INFO, // informational messages + DSS_LOG_LEVEL_COUNT, +} dss_log_level_t; + +typedef enum en_dss_log_id { + DSS_LOG_ID_RUN = 0, + DSS_LOG_ID_DEBUG, + DSS_LOG_ID_COUNT, +} dss_log_id_t; + +status_t srv_device_init(const char *conn_path); + +#endif // __SRV_DEVICE_ADPT_H__ \ No newline at end of file diff --git a/pkg/src/server/srv_instance.h b/pkg/src/server/srv_instance.h index 98b53e6e1bace3326684516a246fe6ceb4c81720..3c6107da981d6525eabecf36438737b5123c4550 100644 --- a/pkg/src/server/srv_instance.h +++ b/pkg/src/server/srv_instance.h @@ -157,6 +157,7 @@ typedef struct st_instance_attr { bool32 disable_var_peek; bool32 enable_cursor_sharing; bool32 enable_use_spm; + bool32 enable_dbstor; } instance_attr_t; typedef struct st_os_run_desc { diff --git a/pkg/src/tse/tse_ddl.c b/pkg/src/tse/tse_ddl.c index 23ded308bcbe26e0175bae19e7c33aa03a092fde..5a0597e9457b74b8bbf0363929cab536216b8ccc 100644 --- a/pkg/src/tse/tse_ddl.c +++ b/pkg/src/tse/tse_ddl.c @@ -993,13 +993,8 @@ static status_t tse_generate_tablespace_path(const char *db_tablespace_name, cha { int ret; char path_prefix[CT_FILE_NAME_BUFFER_SIZE]; - if (cm_dbs_is_enable_dbs() == CT_TRUE) { - PRTS_RETURN_IFERR(snprintf_s(path_prefix, CT_FILE_NAME_BUFFER_SIZE, CT_FILE_NAME_BUFFER_SIZE - 1, - "%s", srv_get_param("SHARED_PATH"))); - } else { - PRTS_RETURN_IFERR(snprintf_s(path_prefix, CT_FILE_NAME_BUFFER_SIZE, CT_FILE_NAME_BUFFER_SIZE - 1, - "%s/data/", g_instance->home)); - } + PRTS_RETURN_IFERR(snprintf_s(path_prefix, CT_FILE_NAME_BUFFER_SIZE, CT_FILE_NAME_BUFFER_SIZE - 1, + "%s", srv_get_param("SHARED_PATH"))); CT_LOG_RUN_INF("[CTC_CREATE_TS] mysql path: %s", path_prefix); int path_prefix_len = path_prefix == NULL ? 0 : strlen(path_prefix); if ((path_prefix_len + strlen(db_tablespace_name)) > path_max_len) { @@ -1010,12 +1005,7 @@ static status_t tse_generate_tablespace_path(const char *db_tablespace_name, cha if (path_prefix == NULL || strlen(path_prefix) == 0) { ret = sprintf_s(db_ts_path, path_max_len, "%s", db_tablespace_name); } else { - if (cm_dbs_is_enable_dbs() == CT_TRUE) { - // for dbstor, remove sep to ensure users can mount from nfs and get correct file desc. - ret = sprintf_s(db_ts_path, path_max_len, "%s%s", path_prefix, db_tablespace_name); - } else { - ret = sprintf_s(db_ts_path, path_max_len, "%s/%s", path_prefix, db_tablespace_name); - } + ret = sprintf_s(db_ts_path, path_max_len, "%s/%s", path_prefix, db_tablespace_name); } knl_securec_check_ss(ret); return CT_SUCCESS;