Comprehensive RDMA Network Technology Guide: From Protocol Principles to Production Practice

Lossless Ethernet Configuration:
  - Priority Flow Control (PFC): Prevents packet loss
  - Explicit Congestion Notification (ECN): Congestion control
  - Data Center Bridging (DCB): Traffic management
  - Enhanced Transmission Selection (ETS): Bandwidth allocation

# MPI over RDMA configuration example
export OMPI_MCA_btl=openib,self
export OMPI_MCA_btl_openib_use_eager_rdma=1
export OMPI_MCA_btl_openib_cpc_include=rdmacm

# Ceph RDMA configuration
[global]
ms_type = async+rdma
ms_rdma_device_name = mlx5_0
ms_rdma_port_num = 1
ms_rdma_gid_index = 0

# PyTorch distributed training configuration
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel

# Using RDMA backend
dist.init_process_group(
    backend='nccl',  # NCCL backend with RDMA support
    init_method='env://',
    world_size=world_size,
    rank=rank
)

# Alibaba Cloud eRDMA configuration
Instance Type: ecs.ebmgn7i.32xlarge
Network Type: Virtual Private Cloud (VPC)
RDMA Network: Enable eRDMA
Bandwidth: 100Gbps
Latency: <10μs

# AWS EFA configuration
export FI_PROVIDER=efa
export FI_EFA_ENABLE_SHM_TRANSFER=1
export FI_EFA_USE_DEVICE_RDMA=1

# TiDB RDMA configuration
[server]
socket = "/tmp/tidb.sock"
[raftstore]
raft-msg-max-batch-size = 1024
raft-msg-flush-interval = "2ms"

# Switch PFC configuration
interface Ethernet1/1
  priority-flow-control mode on
  priority-flow-control priority 3,4

# ECN configuration
interface Ethernet1/1
  ecn
  ecn threshold 1000 10000

# Network card configuration
ethtool -G ens1f0 rx 4096 tx 4096
ethtool -K ens1f0 gro off lro off
ethtool -A ens1f0 autoneg off rx on tx on

# RDMA device configuration
ibv_devinfo
ibv_rc_pingpong -d mlx5_0 -g 0

# Network buffer optimization
cat >> /etc/sysctl.conf << EOF
# RDMA network optimization parameters
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.core.netdev_max_backlog = 5000
net.core.netdev_budget = 600

# Memory management optimization
vm.swappiness = 10
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
vm.dirty_expire_centisecs = 3000
vm.dirty_writeback_centisecs = 500

# Network stack optimization
net.ipv4.tcp_rmem = 4096 262144 134217728
net.ipv4.tcp_wmem = 4096 262144 134217728
net.ipv4.tcp_congestion_control = bbr
net.core.somaxconn = 65535
EOF

# Apply configuration
sysctl -p

# CPU performance mode
echo performance > /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

# Disable CPU power saving
echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo

# Memory huge pages configuration
echo 1024 > /proc/sys/vm/nr_hugepages
echo 'vm.nr_hugepages = 1024' >> /etc/sysctl.conf

# NUMA optimization
echo 0 > /proc/sys/kernel/numa_balancing

# Get network card interrupt information
grep -H . /proc/interrupts | grep mlx5

# Set interrupt affinity (avoid CPU 0)
echo 2 > /proc/irq/24/smp_affinity
echo 4 > /proc/irq/25/smp_affinity
echo 8 > /proc/irq/26/smp_affinity

# Bind RDMA process to specific CPUs
taskset -c 2-7 your_rdma_application

# Network card queue configuration
ethtool -L ens1f0 combined 16
ethtool -L ens1f0 rx 16 tx 16

# Buffer size adjustment
ethtool -G ens1f0 rx 4096 tx 4096

# Disable unnecessary features
ethtool -K ens1f0 gro off lro off tso off gso off
ethtool -K ens1f0 rxhash off

# Flow control configuration
ethtool -A ens1f0 autoneg off rx on tx on
ethtool -s ens1f0 speed 100000 duplex full autoneg off

# Enable PFC (Priority Flow Control)
# Switch configuration example
interface Ethernet1/1
  priority-flow-control mode on
  priority-flow-control priority 3,4
  no shutdown

# Host-side PFC configuration
echo 1 > /sys/class/net/ens1f0/device/sriov_numvfs
echo 0 > /sys/class/net/ens1f0/device/sriov_numvfs

# ECN configuration
echo 1 > /proc/sys/net/ipv4/tcp_ecn

// Memory pre-allocation and pooling
struct memory_pool {
    void **buffers;
    int pool_size;
    int current_index;
    pthread_mutex_t mutex;
};

// Pre-register memory regions
struct ibv_mr *mr = ibv_reg_mr(pd, buffer, size, 
    IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | 
    IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC);

// Memory alignment optimization
void *aligned_buffer = aligned_alloc(4096, buffer_size);

// Optimize QP attributes
struct ibv_qp_init_attr qp_init_attr = {
    .send_cq = send_cq,
    .recv_cq = recv_cq,
    .cap = {
        .max_send_wr = 2048,        // Increase send queue depth
        .max_recv_wr = 2048,        // Increase receive queue depth
        .max_send_sge = 16,         // Support scatter-gather operations
        .max_recv_sge = 16,
        .max_inline_data = 64       // Inline transmission for small messages
    },
    .qp_type = IBV_QPT_RC,
    .sq_sig_all = 0                 // Reduce signaling frequency
};

// Batch operation optimization
struct ibv_send_wr *send_wr_list[16];
struct ibv_send_wr *bad_wr;
int num_wr = 16;

// Batch submit send requests
ibv_post_send(qp, send_wr_list, &bad_wr);

// Use inline data to reduce memory access
struct ibv_send_wr send_wr = {
    .wr_id = 1,
    .next = NULL,
    .sg_list = &sg,
    .num_sge = 1,
    .opcode = IBV_WR_SEND,
    .send_flags = IBV_SEND_INLINE | IBV_SEND_SIGNALED
};

// Use signal batching
if (++send_count % 16 == 0) {
    send_wr.send_flags = IBV_SEND_SIGNALED;
}

# Leaf-spine network design
spine_switches: 4
leaf_switches: 8
servers_per_leaf: 32
oversubscription_ratio: 1:1  # Non-blocking design

# Redundancy design
redundancy_level: 2+1  # 2 active paths + 1 backup
failover_time: <100ms

# ECMP configuration
ip route add default nexthop via 10.0.1.1 dev ens1f0 weight 1 \
    nexthop via 10.0.1.2 dev ens1f0 weight 1

# Flow hashing configuration
echo 1 > /sys/class/net/ens1f0/queues/rx-0/rps_cpus
echo 2 > /sys/class/net/ens1f0/queues/rx-1/rps_cpus

# Firewall rules
iptables -A INPUT -p tcp --dport 18515 -j ACCEPT  # InfiniBand
iptables -A INPUT -p udp --dport 4791 -j ACCEPT   # RoCE

# Access control
echo "192.168.1.0/24" > /etc/rdma/rdma.conf

# 监控系统架构
monitoring_stack:
  metrics_collection:
    - node_exporter: 系统指标
    - prometheus: 时序数据库
    - grafana: 可视化面板
  rdma_specific:
    - rdma_exporter: RDMA专用指标
    - custom_scripts: 自定义监控脚本
  alerting:
    - alertmanager: 告警管理
    - webhook: 告警通知

# CPU和内存使用率
CPU_USAGE=$(top -bn1  grep "Cpu(s)"  awk '{print $2}'  cut -d'%' -f1)
MEMORY_USAGE=$(free  grep Mem  awk '{printf "%.2f", $3/$2 * 100.0}')

# 网络接口统计
NETWORK_STATS=$(cat /proc/net/dev  grep ens1f0)
RX_BYTES=$(echo $NETWORK_STATS  awk '{print $2}')
TX_BYTES=$(echo $NETWORK_STATS  awk '{print $10}')

# 中断统计
INTERRUPT_STATS=$(cat /proc/interrupts  grep mlx5)

# RDMA设备状态
RDMA_DEVICES=$(rdma dev show  grep -c "mlx5")
RDMA_PORTS=$(rdma link show  grep -c "state ACTIVE")

# 队列对状态
QP_COUNT=$(rdma res show qp  wc -l)
QP_ERRORS=$(rdma res show qp  grep -c "state ERROR")

# 内存注册统计
MR_COUNT=$(rdma res show mr  wc -l)
MR_SIZE=$(rdma res show mr  awk '{sum+=$3} END {print sum}')

# 完成队列统计
CQ_COUNT=$(rdma res show cq  wc -l)
CQ_OVERFLOW=$(rdma res show cq  grep -c "overflow")

#!/bin/bash
# rdma_monitor.sh - RDMA实时监控脚本

INTERVAL=5
LOG_FILE="/var/log/rdma_monitor.log"

monitor_rdma() {
    while true; do
        TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
        
        # 获取RDMA设备信息
        DEVICE_INFO=$(rdma dev show  grep mlx5)
        PORT_INFO=$(rdma link show  grep mlx5)
        
        # 获取性能统计
        PERFORMANCE=$(ibv_rc_pingpong -d mlx5_0 -g 0 -n 1000 2>&1  tail -1)
        
        # 记录到日志
        echo "[$TIMESTAMP] Device: $DEVICE_INFO" >> $LOG_FILE
        echo "[$TIMESTAMP] Port: $PORT_INFO" >> $LOG_FILE
        echo "[$TIMESTAMP] Performance: $PERFORMANCE" >> $LOG_FILE
        
        sleep $INTERVAL
    done
}

# 启动监控
monitor_rdma &

# 诊断网络连通性
ping_rdma() {
    local target_ip=$1
    local port=${2:-18515}
    
    # 检查网络连通性
    ping -c 3 $target_ip
    
    # 检查端口连通性
    nc -zv $target_ip $port
    
    # 检查RDMA连接
    ibv_rc_pingpong -d mlx5_0 -g 0 -s 1024 $target_ip
}

# 诊断网络配置
diagnose_network() {
    echo "=== 网络接口状态 ==="
    ip link show  grep -A 5 ens1f0
    
    echo "=== 路由表 ==="
    ip route show
    
    echo "=== ARP表 ==="
    arp -a  grep ens1f0
    
    echo "=== 网络统计 ==="
    cat /proc/net/dev  grep ens1f0
}

# 诊断RDMA设备
diagnose_rdma_device() {
    echo "=== RDMA设备列表 ==="
    rdma dev show
    
    echo "=== 设备详细信息 ==="
    ibv_devinfo -v
    
    echo "=== 端口状态 ==="
    rdma link show
    
    echo "=== 设备统计 ==="
    cat /sys/class/infiniband/mlx5_0/ports/1/counters/*
}

# 重置RDMA设备
reset_rdma_device() {
    echo "重置RDMA设备..."
    
    # 卸载驱动
    modprobe -r mlx5_ib
    modprobe -r mlx5_core
    
    # 重新加载驱动
    modprobe mlx5_core
    modprobe mlx5_ib
    
    # 检查设备状态
    sleep 5
    rdma dev show
}

# 性能基准测试
performance_test() {
    local target_ip=$1
    local test_size=${2:-1048576}  # 1MB
    
    echo "=== 延迟测试 ==="
    ibv_rc_pingpong -d mlx5_0 -g 0 -s 64 -n 1000 $target_ip
    
    echo "=== 带宽测试 ==="
    ib_write_bw -d mlx5_0 -x 1 -s $test_size $target_ip
    
    echo "=== 双向带宽测试 ==="
    ib_write_bw -d mlx5_0 -x 1 -s $test_size -a $target_ip &
    ib_read_bw -d mlx5_0 -x 1 -s $test_size -a $target_ip
}

# 性能问题分析
analyze_performance() {
    echo "=== CPU使用率 ==="
    top -bn1  head -20
    
    echo "=== 内存使用情况 ==="
    free -h
    
    echo "=== 网络中断统计 ==="
    cat /proc/interrupts  grep mlx5
    
    echo "=== 网络队列状态 ==="
    cat /proc/net/softnet_stat  head -5
    
    echo "=== RDMA统计信息 ==="
    cat /sys/class/infiniband/mlx5_0/ports/1/counters/*
}

# 内存诊断
diagnose_memory() {
    echo "=== 系统内存状态 ==="
    free -h
    cat /proc/meminfo  grep -E "(MemTotalMemFreeMemAvailableHugePages)"
    
    echo "=== RDMA内存注册 ==="
    rdma res show mr
    
    echo "=== 内存泄漏检查 ==="
    # 检查内存注册是否正常释放
    for i in {1..10}; do
        echo "测试 $i:"
        rdma res show mr  wc -l
        sleep 1
    done
}

# 内存优化建议
memory_optimization() {
    echo "=== 内存优化建议 ==="
    
    # 检查大页配置
    HUGEPAGES=$(cat /proc/sys/vm/nr_hugepages)
    echo "当前大页数量: $HUGEPAGES"
    
    if [ $HUGEPAGES -lt 1024 ]; then
        echo "建议增加大页数量: echo 1024 > /proc/sys/vm/nr_hugepages"
    fi
    
    # 检查内存碎片
    FRAGMENTATION=$(cat /proc/buddyinfo  awk '{sum+=$2} END {print sum}')
    echo "内存碎片情况: $FRAGMENTATION"
}

#!/bin/bash
# rdma_diagnosis.sh - 自动化RDMA故障诊断

LOG_DIR="/var/log/rdma_diagnosis"
mkdir -p $LOG_DIR

# 创建诊断报告
create_report() {
    local report_file="$LOG_DIR/rdma_diagnosis_$(date +%Y%m%d_%H%M%S).log"
    
    {
        echo "RDMA故障诊断报告 - $(date)"
        echo "=================================="
        
        echo -e "\n1. 系统信息:"
        uname -a
        cat /etc/os-release
        
        echo -e "\n2. 硬件信息:"
        lspci  grep -i mellanox
        lspci  grep -i infiniband
        
        echo -e "\n3. 网络配置:"
        ip addr show
        ip route show
        
        echo -e "\n4. RDMA设备状态:"
        rdma dev show
        rdma link show
        
        echo -e "\n5. 设备详细信息:"
        ibv_devinfo -v
        
        echo -e "\n6. 性能统计:"
        cat /sys/class/infiniband/mlx5_0/ports/1/counters/*
        
        echo -e "\n7. 系统资源:"
        free -h
        df -h
        
        echo -e "\n8. 进程信息:"
        ps aux  grep -E "(rdmaibvmlx5)"
        
    } > $report_file
    
    echo "诊断报告已保存到: $report_file"
}

# 执行诊断
create_report

# prometheus.yml
global:
  scrape_interval: 15s

rule_files:
  - "rdma_rules.yml"

scrape_configs:
  - job_name: 'rdma-nodes'
    static_configs:
      - targets: ['localhost:9100', 'node1:9100', 'node2:9100']
  
  - job_name: 'rdma-metrics'
    static_configs:
      - targets: ['localhost:9400']  # RDMA exporter
    scrape_interval: 5s

# rdma_rules.yml
groups:
- name: rdma_alerts
  rules:
  - alert: RDMADeviceDown
    expr: rdma_device_up == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "RDMA设备离线"
      description: "RDMA设备 {{ $labels.device }} 已离线超过1分钟"
  
  - alert: RDMAHighLatency
    expr: rdma_latency_p99 > 10
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "RDMA延迟过高"
      description: "RDMA P99延迟 {{ $value }}μs 超过阈值"
  
  - alert: RDMAHighErrorRate
    expr: rate(rdma_errors_total[5m]) > 0.01
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "RDMA错误率过高"
      description: "RDMA错误率 {{ $value }} 超过1%"
  
  - alert: RDMAQueueFull
    expr: rdma_queue_utilization > 0.9
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "RDMA队列使用率过高"
      description: "队列 {{ $labels.queue }} 使用率 {{ $value }}% 超过90%"

{
  "dashboard": {
    "title": "RDMA监控仪表板",
    "panels": [
      {
        "title": "RDMA设备状态",
        "type": "stat",
        "targets": [
          {
            "expr": "rdma_device_up",
            "legendFormat": "设备状态"
          }
        ]
      },
      {
        "title": "RDMA延迟分布",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.50, rdma_latency_seconds_bucket)",
            "legendFormat": "P50延迟"
          },
          {
            "expr": "histogram_quantile(0.95, rdma_latency_seconds_bucket)",
            "legendFormat": "P95延迟"
          },
          {
            "expr": "histogram_quantile(0.99, rdma_latency_seconds_bucket)",
            "legendFormat": "P99延迟"
          }
        ]
      },
      {
        "title": "RDMA带宽使用率",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(rdma_bytes_total[5m])",
            "legendFormat": "带宽使用率"
          }
        ]
      }
    ]
  }
}

#!/bin/bash
# rdma_benchmark.sh - RDMA性能基准测试

BENCHMARK_DIR="/var/log/rdma_benchmark"
mkdir -p $BENCHMARK_DIR

# 测试参数
TEST_SIZES=(64 1024 4096 16384 65536 262144 1048576)  # 字节
TEST_ITERATIONS=1000
TARGET_IP="192.168.1.100"

run_benchmark() {
    local test_name=$1
    local test_cmd=$2
    local result_file="$BENCHMARK_DIR/${test_name}_$(date +%Y%m%d_%H%M%S).log"
    
    echo "运行测试: $test_name"
    echo "命令: $test_cmd"
    echo "结果文件: $result_file"
    
    eval $test_cmd > $result_file 2>&1
    
    if [ $? -eq 0 ]; then
        echo "测试完成: $test_name"
    else
        echo "测试失败: $test_name"
    fi
}

# 延迟测试
run_latency_test() {
    for size in "${TEST_SIZES[@]}"; do
        run_benchmark "latency_${size}B" \
            "ibv_rc_pingpong -d mlx5_0 -g 0 -s $size -n $TEST_ITERATIONS $TARGET_IP"
    done
}

# 带宽测试
run_bandwidth_test() {
    for size in "${TEST_SIZES[@]}"; do
        run_benchmark "bandwidth_${size}B" \
            "ib_write_bw -d mlx5_0 -x 1 -s $size $TARGET_IP"
    done
}

# 双向带宽测试
run_dual_bandwidth_test() {
    for size in "${TEST_SIZES[@]}"; do
        run_benchmark "dual_bandwidth_${size}B" \
            "ib_write_bw -d mlx5_0 -x 1 -s $size -a $TARGET_IP & ib_read_bw -d mlx5_0 -x 1 -s $size -a $TARGET_IP"
    done
}

# 执行所有测试
echo "开始RDMA性能基准测试..."
run_latency_test
run_bandwidth_test
run_dual_bandwidth_test
echo "所有测试完成，结果保存在: $BENCHMARK_DIR"

#!/bin/bash
# hardware_compatibility_check.sh

check_hardware_compatibility() {
    echo "=== 硬件兼容性检查 ==="
    
    # 检查CPU架构
    ARCH=$(uname -m)
    echo "CPU架构: $ARCH"
    
    # 检查网卡型号
    NETWORK_CARDS=$(lspci  grep -i "ethernet\infiniband\mellanox")
    echo "网络设备:"
    echo "$NETWORK_CARDS"
    
    # 检查内存大小
    MEMORY_GB=$(free -g  grep Mem  awk '{print $2}')
    echo "内存大小: ${MEMORY_GB}GB"
    
    # 检查NUMA拓扑
    echo "NUMA拓扑:"
    numactl --hardware
    
    # 检查PCIe插槽
    echo "PCIe设备:"
    lspci -tv  grep -E "PCIeEthernetInfiniBand"
}

# 执行检查
check_hardware_compatibility

#!/bin/bash
# system_preparation.sh

prepare_system() {
    echo "=== 系统环境准备 ==="
    
    # 更新系统
    yum update -y  apt update && apt upgrade -y
    
    # 安装必要软件包
    yum install -y rdma-core infiniband-diags perftest  \
    apt install -y rdma-core infiniband-diags perftest
    
    # 安装开发工具
    yum groupinstall -y "Development Tools"  \
    apt install -y build-essential
    
    # 配置内核参数
    configure_kernel_params
    
    # 配置网络
    configure_network
    
    # 配置防火墙
    configure_firewall
}

configure_kernel_params() {
    echo "配置内核参数..."
    
    cat >> /etc/sysctl.conf << EOF
# RDMA优化参数
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.core.netdev_max_backlog = 5000
net.core.netdev_budget = 600
vm.swappiness = 10
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
EOF
    
    sysctl -p
}

configure_network() {
    echo "配置网络..."
    
    # 禁用网络管理服务
    systemctl stop NetworkManager
    systemctl disable NetworkManager
    
    # 配置静态IP
    cat > /etc/sysconfig/network-scripts/ifcfg-ens1f0 << EOF
TYPE=Ethernet
BOOTPROTO=static
NAME=ens1f0
DEVICE=ens1f0
ONBOOT=yes
IPADDR=192.168.1.100
NETMASK=255.255.255.0
GATEWAY=192.168.1.1
EOF
}

configure_firewall() {
    echo "配置防火墙..."
    
    # 开放RDMA端口
    firewall-cmd --permanent --add-port=18515/tcp  # InfiniBand
    firewall-cmd --permanent --add-port=4791/udp   # RoCE
    firewall-cmd --reload
}

# 执行准备
prepare_system

#!/bin/bash
# rdma_deployment.sh

DEPLOYMENT_LOG="/var/log/rdma_deployment.log"

deploy_rdma() {
    local node_type=$1  # "server" or "client"
    local target_ip=$2
    
    echo "开始部署RDMA - 节点类型: $node_type"  tee -a $DEPLOYMENT_LOG
    
    # 1. 检查硬件
    check_hardware
    
    # 2. 安装驱动
    install_drivers
    
    # 3. 配置网络
    configure_rdma_network
    
    # 4. 测试连接
    test_connection $target_ip
    
    # 5. 性能验证
    performance_validation $target_ip
    
    echo "RDMA部署完成"  tee -a $DEPLOYMENT_LOG
}

check_hardware() {
    echo "检查硬件兼容性..."  tee -a $DEPLOYMENT_LOG
    
    # 检查网卡
    if ! lspci  grep -i mellanox; then
        echo "错误: 未检测到Mellanox网卡"  tee -a $DEPLOYMENT_LOG
        exit 1
    fi
    
    # 检查内存
    MEMORY_GB=$(free -g  grep Mem  awk '{print $2}')
    if [ $MEMORY_GB -lt 16 ]; then
        echo "警告: 内存不足16GB，可能影响性能"  tee -a $DEPLOYMENT_LOG
    fi
}

install_drivers() {
    echo "安装RDMA驱动..."  tee -a $DEPLOYMENT_LOG
    
    # 安装Mellanox驱动
    if [ -f "/opt/mellanox/mlnxofedinstall/mlnxofedinstall" ]; then
        /opt/mellanox/mlnxofedinstall/mlnxofedinstall --auto
    else
        # 使用系统包管理器安装
        yum install -y rdma-core  apt install -y rdma-core
    fi
    
    # 加载驱动
    modprobe mlx5_core
    modprobe mlx5_ib
    
    # 检查驱动状态
    if ! rdma dev show  grep mlx5; then
        echo "错误: RDMA驱动加载失败"  tee -a $DEPLOYMENT_LOG
        exit 1
    fi
}

configure_rdma_network() {
    echo "配置RDMA网络..."  tee -a $DEPLOYMENT_LOG
    
    # 配置RoCE
    if [ -f "/sys/class/net/ens1f0/device/sriov_numvfs" ]; then
        echo 1 > /sys/class/net/ens1f0/device/sriov_numvfs
        echo 0 > /sys/class/net/ens1f0/device/sriov_numvfs
    fi
    
    # 配置网卡参数
    ethtool -L ens1f0 combined 16
    ethtool -G ens1f0 rx 4096 tx 4096
    ethtool -K ens1f0 gro off lro off tso off gso off
}

test_connection() {
    local target_ip=$1
    echo "测试RDMA连接..."  tee -a $DEPLOYMENT_LOG
    
    # 基本连通性测试
    if ! ping -c 3 $target_ip; then
        echo "错误: 网络连通性测试失败"  tee -a $DEPLOYMENT_LOG
        exit 1
    fi
    
    # RDMA连接测试
    if ! ibv_rc_pingpong -d mlx5_0 -g 0 -s 1024 $target_ip; then
        echo "错误: RDMA连接测试失败"  tee -a $DEPLOYMENT_LOG
        exit 1
    fi
}

performance_validation() {
    local target_ip=$1
    echo "性能验证..."  tee -a $DEPLOYMENT_LOG
    
    # 延迟测试
    LATENCY=$(ibv_rc_pingpong -d mlx5_0 -g 0 -s 64 -n 1000 $target_ip 2>&1  grep "latency"  awk '{print $NF}')
    echo "延迟测试结果: $LATENCY"  tee -a $DEPLOYMENT_LOG
    
    # 带宽测试
    BANDWIDTH=$(ib_write_bw -d mlx5_0 -x 1 -s 1048576 $target_ip 2>&1  grep "BW"  awk '{print $2}')
    echo "带宽测试结果: $BANDWIDTH"  tee -a $DEPLOYMENT_LOG
}

# 主函数
main() {
    if [ $# -lt 2 ]; then
        echo "用法: $0 <serverclient> <target_ip>"
        exit 1
    fi
    
    deploy_rdma $1 $2
}

# 执行部署
main "$@"

#!/bin/bash
# rdma_maintenance.sh

MAINTENANCE_LOG="/var/log/rdma_maintenance.log"

daily_maintenance() {
    echo "=== RDMA日常维护 - $(date) ==="  tee -a $MAINTENANCE_LOG
    
    # 1. 检查设备状态
    check_device_status
    
    # 2. 检查性能指标
    check_performance_metrics
    
    # 3. 检查错误日志
    check_error_logs
    
    # 4. 清理临时文件
    cleanup_temp_files
    
    # 5. 生成维护报告
    generate_maintenance_report
}

check_device_status() {
    echo "检查RDMA设备状态..."  tee -a $MAINTENANCE_LOG
    
    # 设备列表
    DEVICES=$(rdma dev show  grep mlx5  wc -l)
    echo "活跃RDMA设备数量: $DEVICES"  tee -a $MAINTENANCE_LOG
    
    # 端口状态
    PORTS=$(rdma link show  grep "state ACTIVE"  wc -l)
    echo "活跃端口数量: $PORTS"  tee -a $MAINTENANCE_LOG
    
    # 队列对状态
    QP_COUNT=$(rdma res show qp  wc -l)
    QP_ERRORS=$(rdma res show qp  grep "state ERROR"  wc -l)
    echo "队列对总数: $QP_COUNT, 错误数量: $QP_ERRORS"  tee -a $MAINTENANCE_LOG
}

check_performance_metrics() {
    echo "检查性能指标..."  tee -a $MAINTENANCE_LOG
    
    # 网络统计
    NETWORK_STATS=$(cat /proc/net/dev  grep ens1f0)
    RX_BYTES=$(echo $NETWORK_STATS  awk '{print $2}')
    TX_BYTES=$(echo $NETWORK_STATS  awk '{print $10}')
    echo "接收字节数: $RX_BYTES, 发送字节数: $TX_BYTES"  tee -a $MAINTENANCE_LOG
    
    # 中断统计
    INTERRUPT_COUNT=$(cat /proc/interrupts  grep mlx5  awk '{sum+=$2} END {print sum}')
    echo "网卡中断次数: $INTERRUPT_COUNT"  tee -a $MAINTENANCE_LOG
}

check_error_logs() {
    echo "检查错误日志..."  tee -a $MAINTENANCE_LOG
    
    # 系统日志中的RDMA错误
    RDMA_ERRORS=$(journalctl -u rdma --since "1 day ago"  grep -i error  wc -l)
    echo "过去24小时RDMA错误数量: $RDMA_ERRORS"  tee -a $MAINTENANCE_LOG
    
    # 内核日志中的网络错误
    NETWORK_ERRORS=$(dmesg  grep -i "network\ethernet\rdma"  grep -i error  wc -l)
    echo "内核日志中网络错误数量: $NETWORK_ERRORS"  tee -a $MAINTENANCE_LOG
}

cleanup_temp_files() {
    echo "清理临时文件..."  tee -a $MAINTENANCE_LOG
    
    # 清理RDMA测试文件
    find /tmp -name "rdma_*" -mtime +7 -delete 2>/dev/null
    
    # 清理日志文件
    find /var/log -name "rdma_*.log" -mtime +30 -delete 2>/dev/null
    
    echo "临时文件清理完成"  tee -a $MAINTENANCE_LOG
}

generate_maintenance_report() {
    local report_file="/var/log/rdma_maintenance_report_$(date +%Y%m%d).log"
    
    {
        echo "RDMA维护报告 - $(date)"
        echo "================================"
        
        echo -e "\n设备状态:"
        rdma dev show
        rdma link show
        
        echo -e "\n性能统计:"
        cat /sys/class/infiniband/mlx5_0/ports/1/counters/*
        
        echo -e "\n系统资源:"
        free -h
        df -h
        
    } > $report_file
    
    echo "维护报告已生成: $report_file"  tee -a $MAINTENANCE_LOG
}

# 执行日常维护
daily_maintenance

#!/bin/bash
# rdma_recovery.sh

RECOVERY_LOG="/var/log/rdma_recovery.log"

recover_rdma() {
    local issue_type=$1
    
    echo "开始RDMA故障恢复 - 问题类型: $issue_type"  tee -a $RECOVERY_LOG
    
    case $issue_type in
        "device_down")
            recover_device_down
            ;;
        "performance_degradation")
            recover_performance_issues
            ;;
        "connection_failure")
            recover_connection_issues
            ;;
        "memory_issues")
            recover_memory_issues
            ;;
        *)
            echo "未知问题类型: $issue_type"  tee -a $RECOVERY_LOG
            exit 1
            ;;
    esac
}

recover_device_down() {
    echo "恢复RDMA设备..."  tee -a $RECOVERY_LOG
    
    # 1. 检查设备状态
    if ! rdma dev show  grep mlx5; then
        echo "设备未检测到，尝试重新加载驱动..."  tee -a $RECOVERY_LOG
        
        # 卸载驱动
        modprobe -r mlx5_ib
        modprobe -r mlx5_core
        
        # 重新加载驱动
        modprobe mlx5_core
        modprobe mlx5_ib
        
        # 等待设备初始化
        sleep 10
        
        # 检查设备状态
        if rdma dev show  grep mlx5; then
            echo "设备恢复成功"  tee -a $RECOVERY_LOG
        else
            echo "设备恢复失败，需要重启系统"  tee -a $RECOVERY_LOG
            return 1
        fi
    fi
}

recover_performance_issues() {
    echo "恢复性能问题..."  tee -a $RECOVERY_LOG
    
    # 1. 检查CPU使用率
    CPU_USAGE=$(top -bn1  grep "Cpu(s)"  awk '{print $2}'  cut -d'%' -f1)
    if (( $(echo "$CPU_USAGE > 80"  bc -l) )); then
        echo "CPU使用率过高: $CPU_USAGE%，尝试优化..."  tee -a $RECOVERY_LOG
        
        # 调整CPU调度策略
        echo performance > /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
    fi
    
    # 2. 检查内存使用率
    MEMORY_USAGE=$(free  grep Mem  awk '{printf "%.2f", $3/$2 * 100.0}')
    if (( $(echo "$MEMORY_USAGE > 90"  bc -l) )); then
        echo "内存使用率过高: $MEMORY_USAGE%，尝试清理..."  tee -a $RECOVERY_LOG
        
        # 清理页面缓存
        echo 3 > /proc/sys/vm/drop_caches
    fi
    
    # 3. 检查网络队列
    QUEUE_DROPS=$(cat /proc/net/softnet_stat  awk '{sum+=$3} END {print sum}')
    if [ $QUEUE_DROPS -gt 0 ]; then
        echo "检测到网络丢包: $QUEUE_DROPS，调整队列参数..."  tee -a $RECOVERY_LOG
        
        # 调整网络队列参数
        echo 5000 > /proc/sys/net/core/netdev_max_backlog
        echo 600 > /proc/sys/net/core/netdev_budget
    fi
}

recover_connection_issues() {
    echo "恢复连接问题..."  tee -a $RECOVERY_LOG
    
    # 1. 检查网络接口状态
    if ! ip link show ens1f0  grep "state UP"; then
        echo "网络接口未启动，尝试启动..."  tee -a $RECOVERY_LOG
        ip link set ens1f0 up
    fi
    
    # 2. 检查路由表
    if ! ip route show  grep default; then
        echo "默认路由缺失，尝试添加..."  tee -a $RECOVERY_LOG
        ip route add default via 192.168.1.1 dev ens1f0
    fi
    
    # 3. 检查ARP表
    ARP_ENTRIES=$(arp -a  grep ens1f0  wc -l)
    if [ $ARP_ENTRIES -eq 0 ]; then
        echo "ARP表为空，尝试刷新..."  tee -a $RECOVERY_LOG
        arp -d -a
        ping -c 3 192.168.1.1
    fi
}

recover_memory_issues() {
    echo "恢复内存问题..."  tee -a $RECOVERY_LOG
    
    # 1. 检查内存注册
    MR_COUNT=$(rdma res show mr  wc -l)
    if [ $MR_COUNT -gt 1000 ]; then
        echo "内存注册过多: $MR_COUNT，可能存在内存泄漏..."  tee -a $RECOVERY_LOG
        
        # 重启RDMA服务
        systemctl restart rdma
    fi
    
    # 2. 检查大页配置
    HUGEPAGES=$(cat /proc/sys/vm/nr_hugepages)
    if [ $HUGEPAGES -lt 1024 ]; then
        echo "大页数量不足: $HUGEPAGES，尝试增加..."  tee -a $RECOVERY_LOG
        echo 1024 > /proc/sys/vm/nr_hugepages
    fi
}

# 主函数
main() {
    if [ $# -ne 1 ]; then
        echo "用法: $0 <device_downperformance_degradationconnection_failurememory_issues>"
        exit 1
    fi
    
    recover_rdma $1
}

# 执行恢复
main "$@"

#!/bin/bash
# rdma_capacity_planning.sh

CAPACITY_LOG="/var/log/rdma_capacity.log"

capacity_analysis() {
    echo "=== RDMA容量分析 - $(date) ==="  tee -a $CAPACITY_LOG
    
    # 1. 当前资源使用情况
    analyze_current_usage
    
    # 2. 性能基准测试
    run_performance_benchmark
    
    # 3. 容量预测
    predict_capacity_needs
    
    # 4. 生成容量报告
    generate_capacity_report
}

analyze_current_usage() {
    echo "分析当前资源使用情况..."  tee -a $CAPACITY_LOG
    
    # CPU使用率
    CPU_USAGE=$(top -bn1  grep "Cpu(s)"  awk '{print $2}'  cut -d'%' -f1)
    echo "CPU使用率: $CPU_USAGE%"  tee -a $CAPACITY_LOG
    
    # 内存使用率
    MEMORY_USAGE=$(free  grep Mem  awk '{printf "%.2f", $3/$2 * 100.0}')
    echo "内存使用率: $MEMORY_USAGE%"  tee -a $CAPACITY_LOG
    
    # 网络带宽使用率
    NETWORK_STATS=$(cat /proc/net/dev  grep ens1f0)
    RX_BYTES=$(echo $NETWORK_STATS  awk '{print $2}')
    TX_BYTES=$(echo $NETWORK_STATS  awk '{print $10}')
    TOTAL_BYTES=$((RX_BYTES + TX_BYTES))
    echo "网络总流量: $TOTAL_BYTES 字节"  tee -a $CAPACITY_LOG
    
    # RDMA资源使用情况
    QP_COUNT=$(rdma res show qp  wc -l)
    MR_COUNT=$(rdma res show mr  wc -l)
    CQ_COUNT=$(rdma res show cq  wc -l)
    echo "RDMA资源: QP=$QP_COUNT, MR=$MR_COUNT, CQ=$CQ_COUNT"  tee -a $CAPACITY_LOG
}

run_performance_benchmark() {
    echo "运行性能基准测试..."  tee -a $CAPACITY_LOG
    
    # 延迟测试
    LATENCY_RESULT=$(ibv_rc_pingpong -d mlx5_0 -g 0 -s 64 -n 1000 2>&1  grep "latency"  awk '{print $NF}')
    echo "延迟测试结果: $LATENCY_RESULT"  tee -a $CAPACITY_LOG
    
    # 带宽测试
    BANDWIDTH_RESULT=$(ib_write_bw -d mlx5_0 -x 1 -s 1048576 2>&1  grep "BW"  awk '{print $2}')
    echo "带宽测试结果: $BANDWIDTH_RESULT"  tee -a $CAPACITY_LOG
}

predict_capacity_needs() {
    echo "预测容量需求..."  tee -a $CAPACITY_LOG
    
    # 基于历史数据预测
    # 这里可以集成更复杂的预测算法
    
    # 简单的线性预测
    CURRENT_LOAD=$(echo "scale=2; $CPU_USAGE + $MEMORY_USAGE"  bc)
    PREDICTED_LOAD=$(echo "scale=2; $CURRENT_LOAD * 1.2"  bc)
    
    echo "当前负载: $CURRENT_LOAD%"  tee -a $CAPACITY_LOG
    echo "预测负载: $PREDICTED_LOAD%"  tee -a $CAPACITY_LOG
    
    # 容量建议
    if (( $(echo "$PREDICTED_LOAD > 80"  bc -l) )); then
        echo "建议: 需要扩容"  tee -a $CAPACITY_LOG
    elif (( $(echo "$PREDICTED_LOAD > 60"  bc -l) )); then
        echo "建议: 监控负载变化"  tee -a $CAPACITY_LOG
    else
        echo "建议: 当前容量充足"  tee -a $CAPACITY_LOG
    fi
}

generate_capacity_report() {
    local report_file="/var/log/rdma_capacity_report_$(date +%Y%m%d).log"
    
    {
        echo "RDMA容量规划报告 - $(date)"
        echo "=================================="
        
        echo -e "\n当前资源使用情况:"
        echo "CPU使用率: $CPU_USAGE%"
        echo "内存使用率: $MEMORY_USAGE%"
        echo "网络总流量: $TOTAL_BYTES 字节"
        
        echo -e "\nRDMA资源使用情况:"
        echo "队列对数量: $QP_COUNT"
        echo "内存注册数量: $MR_COUNT"
        echo "完成队列数量: $CQ_COUNT"
        
        echo -e "\n性能基准测试:"
        echo "延迟: $LATENCY_RESULT"
        echo "带宽: $BANDWIDTH_RESULT"
        
        echo -e "\n容量预测:"
        echo "当前负载: $CURRENT_LOAD%"
        echo "预测负载: $PREDICTED_LOAD%"
        
    } > $report_file
    
    echo "容量报告已生成: $report_file"  tee -a $CAPACITY_LOG
}

# 执行容量分析
capacity_analysis