# Minimum hardware requirements per node
# MON nodes: 3 nodes minimum
# - CPU: 2+ cores
# - RAM: 4GB minimum
# - Network: 1GbE minimum
# - Disk: 50GB for OS

# OSD nodes: 3+ nodes
# - CPU: 1 core per OSD
# - RAM: 4GB + 1GB per OSD
# - Network: 10GbE recommended
# - Disk: Multiple drives for OSDs

# MDS nodes (for CephFS): 2+ nodes
# - CPU: 4+ cores
# - RAM: 8GB minimum
# - Network: 10GbE

# Network architecture
# Public network: Client access
# Cluster network: OSD replication

# Register SUSE Enterprise Storage
sudo SUSEConnect -p ses/7/x86_64 -r REGISTRATION_CODE

# Install ceph-salt
sudo zypper install ceph-salt

# Initialize salt master
sudo systemctl enable salt-master
sudo systemctl start salt-master

# Configure salt minions on all nodes
sudo zypper install salt-minion
echo "master: ceph-admin.example.com" > /etc/salt/minion
sudo systemctl enable salt-minion
sudo systemctl start salt-minion

# Accept salt keys on master
sudo salt-key -L
sudo salt-key -A -y

# Initialize ceph-salt configuration
sudo ceph-salt config /cluster/minions add *.example.com
sudo ceph-salt config /ceph_cluster/minions/all add *.example.com

# Configure roles
sudo ceph-salt config /ceph_cluster/roles/admin add ceph-admin.example.com
sudo ceph-salt config /ceph_cluster/roles/bootstrap add mon1.example.com
sudo ceph-salt config /ceph_cluster/roles/mon add mon[1-3].example.com
sudo ceph-salt config /ceph_cluster/roles/mgr add mon[1-3].example.com
sudo ceph-salt config /ceph_cluster/roles/osd add osd[1-6].example.com

# Configure SSH key
sudo ceph-salt config /ssh generate
sudo ceph-salt config /ssh/private_key use_existing

# Configure time server
sudo ceph-salt config /time_server/server_hostname set ntp.example.com
sudo ceph-salt config /time_server/external_servers add pool.ntp.org

# Deploy Ceph cluster
sudo ceph-salt apply

# Verify cluster status
sudo ceph -s
sudo ceph health detail

# Create replicated pool
sudo ceph osd pool create rbd-pool 128 128 replicated
sudo ceph osd pool set rbd-pool size 3
sudo ceph osd pool set rbd-pool min_size 2

# Create erasure coded pool
sudo ceph osd erasure-code-profile set ec-profile \
  k=4 m=2 \
  plugin=jerasure \
  technique=reed_sol_van

sudo ceph osd pool create ec-pool 128 128 erasure ec-profile
sudo ceph osd pool set ec-pool allow_ec_overwrites true

# Configure pool quotas
sudo ceph osd pool set-quota rbd-pool max_objects 1000000
sudo ceph osd pool set-quota rbd-pool max_bytes 10737418240000  # 10TB

# Enable pool applications
sudo ceph osd pool application enable rbd-pool rbd
sudo ceph osd pool application enable ec-pool rgw

# Create placement groups autoscaler
sudo ceph osd pool set rbd-pool pg_autoscale_mode on
sudo ceph osd pool set ec-pool pg_autoscale_mode on

# Create RBD image
sudo rbd create --size 10G --pool rbd-pool disk01

# List RBD images
sudo rbd ls rbd-pool
sudo rbd info rbd-pool/disk01

# Enable RBD features
sudo rbd feature enable rbd-pool/disk01 exclusive-lock object-map fast-diff

# Create RBD snapshot
sudo rbd snap create rbd-pool/disk01@snapshot1
sudo rbd snap ls rbd-pool/disk01

# Clone RBD image
sudo rbd snap protect rbd-pool/disk01@snapshot1
sudo rbd clone rbd-pool/disk01@snapshot1 rbd-pool/disk01-clone

# Map RBD to kernel device
sudo rbd map rbd-pool/disk01
sudo rbd showmapped

# Create filesystem and mount
sudo mkfs.xfs /dev/rbd0
sudo mkdir /mnt/ceph-disk
sudo mount /dev/rbd0 /mnt/ceph-disk

# Configure automatic mapping at boot
cat > /etc/ceph/rbdmap << 'EOF'
# RBD device mappings
rbd-pool/disk01 id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
EOF

sudo systemctl enable rbdmap

# Deploy MDS servers
sudo ceph orch apply mds cephfs --placement="3 mds1.example.com mds2.example.com mds3.example.com"

# Create CephFS
sudo ceph osd pool create cephfs_data 64
sudo ceph osd pool create cephfs_metadata 16
sudo ceph fs new cephfs cephfs_metadata cephfs_data

# Configure MDS
sudo ceph fs set cephfs max_mds 2
sudo ceph fs set cephfs standby_count_wanted 1

# Mount CephFS using kernel driver
sudo mkdir /mnt/cephfs
sudo mount -t ceph mon1.example.com:6789:/ /mnt/cephfs \
  -o name=admin,secret=$(ceph auth get-key client.admin)

# Mount CephFS using FUSE
sudo ceph-fuse /mnt/cephfs \
  --id admin \
  --conf /etc/ceph/ceph.conf \
  --keyring /etc/ceph/ceph.client.admin.keyring

# Configure CephFS quotas
sudo setfattr -n ceph.quota.max_bytes -v 10737418240 /mnt/cephfs/project1  # 10GB
sudo setfattr -n ceph.quota.max_files -v 100000 /mnt/cephfs/project1

# CephFS snapshots
sudo mkdir /mnt/cephfs/.snap/backup-$(date +%Y%m%d)

# Deploy RadosGW
sudo ceph orch apply rgw myrgw --placement="2 rgw1.example.com rgw2.example.com"

# Create RGW user
sudo radosgw-admin user create \
  --uid=testuser \
  --display-name="Test User" \
  --email=test@example.com

# Create S3 credentials
sudo radosgw-admin key create \
  --uid=testuser \
  --key-type=s3 \
  --gen-access-key \
  --gen-secret

# Configure RGW for S3
cat > /etc/ceph/ceph.conf << 'EOF'
[client.rgw.myrgw]
rgw_frontends = beast port=80
rgw_dns_name = s3.example.com
rgw_enable_usage_log = true
rgw_usage_log_tick_interval = 30
rgw_usage_log_flush_threshold = 1024
rgw_enable_ops_log = true
rgw_ops_log_rados = true
rgw_enable_apis = s3, swift
EOF

# Test S3 access
cat > test-s3.py << 'EOF'
import boto3

s3 = boto3.client('s3',
    endpoint_url='http://s3.example.com',
    aws_access_key_id='ACCESS_KEY',
    aws_secret_access_key='SECRET_KEY')

# Create bucket
s3.create_bucket(Bucket='test-bucket')

# Upload object
s3.put_object(Bucket='test-bucket', Key='test.txt', Body=b'Hello Ceph!')

# List objects
response = s3.list_objects(Bucket='test-bucket')
for obj in response.get('Contents', []):
    print(obj['Key'])
EOF

python3 test-s3.py

# Install NFS HA components
sudo zypper install nfs-kernel-server drbd-utils corosync pacemaker

# Configure shared storage with DRBD
cat > /etc/drbd.d/nfs-data.res << 'EOF'
resource nfs-data {
  device /dev/drbd0;
  disk /dev/sdb1;
  meta-disk internal;

  on nfs1.example.com {
    address 10.0.0.10:7789;
  }

  on nfs2.example.com {
    address 10.0.0.11:7789;
  }

  net {
    protocol C;
    allow-two-primaries no;
    after-sb-0pri discard-zero-changes;
    after-sb-1pri discard-secondary;
  }
}
EOF

# Initialize DRBD
sudo drbdadm create-md nfs-data
sudo drbdadm up nfs-data
sudo drbdadm primary --force nfs-data  # On primary node only

# Create filesystem
sudo mkfs.xfs /dev/drbd0
sudo mkdir /srv/nfs
sudo mount /dev/drbd0 /srv/nfs

# Configure Pacemaker resources
sudo crm configure << 'EOF'
primitive drbd_nfs ocf:linbit:drbd \
  params drbd_resource="nfs-data" \
  op monitor interval="10s"

ms ms_drbd_nfs drbd_nfs \
  meta master-max="1" master-node-max="1" \
  clone-max="2" clone-node-max="1" notify="true"

primitive fs_nfs Filesystem \
  params device="/dev/drbd0" directory="/srv/nfs" fstype="xfs" \
  op monitor interval="10s"

primitive nfs_server systemd:nfs-server \
  op monitor interval="10s"

primitive vip_nfs IPaddr2 \
  params ip="192.168.1.100" cidr_netmask="24" \
  op monitor interval="10s"

group grp_nfs fs_nfs nfs_server vip_nfs

colocation col_nfs_on_drbd inf: grp_nfs ms_drbd_nfs:Master
order ord_drbd_before_nfs inf: ms_drbd_nfs:promote grp_nfs:start

commit
EOF

# Configure NFS exports
cat > /etc/exports << 'EOF'
# Highly Available NFS Exports
/srv/nfs/share1 192.168.1.0/24(rw,sync,no_root_squash,no_subtree_check)
/srv/nfs/share2 192.168.1.0/24(rw,sync,root_squash,no_subtree_check)
/srv/nfs/homes  192.168.1.0/24(rw,sync,root_squash,subtree_check)
EOF

# Create export directories
mkdir -p /srv/nfs/{share1,share2,homes}
chmod 755 /srv/nfs/share1
chmod 755 /srv/nfs/share2
chmod 755 /srv/nfs/homes

# Configure NFSv4 ID mapping
cat > /etc/idmapd.conf << 'EOF'
[General]
Domain = example.com

[Mapping]
Nobody-User = nobody
Nobody-Group = nobody

[Translation]
Method = nsswitch
EOF

# Export configuration for specific use cases
cat > /etc/exports.d/databases.exports << 'EOF'
# Database storage with specific options
/srv/nfs/databases 192.168.1.0/24(rw,sync,no_root_squash,no_subtree_check,anonuid=1000,anongid=1000)
EOF

cat > /etc/exports.d/vmware.exports << 'EOF'
# VMware datastore
/srv/nfs/vmware 192.168.1.0/24(rw,sync,no_root_squash,no_subtree_check,insecure)
EOF

# Apply exports
exportfs -ra
exportfs -v

# NFS server optimization
cat > /etc/sysconfig/nfs << 'EOF'
# Number of NFS server threads
RPCNFSDCOUNT="32"

# NFS protocol versions
RPCNFSDARGS="-N 2 -N 3 -V 4 -V 4.1 -V 4.2"

# Mount protocol versions
RPCMOUNTDOPTS="-N 2 -N 3"

# Disable NFSv4 callback
#RPCIDMAPDARGS="-S"

# TCP settings
RPCSVCGSSDARGS="-n"
EOF

# Kernel tuning for NFS
cat >> /etc/sysctl.d/nfs-performance.conf << 'EOF'
# NFS Performance Tuning
net.core.rmem_default = 262144
net.core.rmem_max = 16777216
net.core.wmem_default = 262144
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 262144 16777216
net.ipv4.tcp_wmem = 4096 262144 16777216
net.ipv4.tcp_no_metrics_save = 1
net.ipv4.tcp_congestion_control = cubic
sunrpc.tcp_slot_table_entries = 128
EOF

sudo sysctl -p /etc/sysctl.d/nfs-performance.conf

# Client mount options for performance
cat > /etc/fstab << 'EOF'
# Optimized NFS mounts
nfs-server:/srv/nfs/share1 /mnt/share1 nfs4 rw,hard,intr,rsize=1048576,wsize=1048576,timeo=600,retrans=2,_netdev 0 0
nfs-server:/srv/nfs/databases /mnt/databases nfs4 rw,hard,intr,rsize=65536,wsize=65536,timeo=600,retrans=2,actimeo=0,_netdev 0 0
EOF

# Install multipath tools
sudo zypper install multipath-tools

# Discover FC/iSCSI devices
sudo multipath -ll
sudo multipathd show paths

# Configure multipath
cat > /etc/multipath.conf << 'EOF'
defaults {
    user_friendly_names yes
    polling_interval 10
    path_selector "round-robin 0"
    path_grouping_policy multibus
    path_checker readsector0
    rr_min_io 100
    max_fds 8192
    rr_weight priorities
    failback immediate
    no_path_retry fail
    features "0"
}

blacklist {
    wwid 26353900f02796769
    devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
    devnode "^hd[a-z]"
    devnode "^cciss!c[0-9]d[0-9]*"
}

multipaths {
    multipath {
        wwid 3600508b4000156d700012000000b0000
        alias yellow
        path_grouping_policy multibus
        path_checker readsector0
        path_selector "round-robin 0"
        failback manual
        rr_weight priorities
        no_path_retry 5
    }
}

devices {
    device {
        vendor "NETAPP"
        product "LUN.*"
        path_grouping_policy group_by_prio
        prio "alua"
        path_checker tur
        failback immediate
        path_selector "round-robin 0"
        hardware_handler "1 alua"
        rr_weight uniform
        rr_min_io 128
        getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
    }
    device {
        vendor "EMC"
        product "SYMMETRIX"
        path_grouping_policy multibus
        path_checker tur
        checker_timeout 30
        hardware_handler "1 emc"
        prio emc
        failback immediate
        rr_weight priorities
        no_path_retry 6
        rr_min_io 1000
    }
}
EOF

# Start multipath service
sudo systemctl enable multipathd
sudo systemctl start multipathd

# Verify multipath status
sudo multipath -ll
sudo multipathd show maps

# Install iSCSI initiator
sudo zypper install open-iscsi

# Configure iSCSI initiator
sudo iscsiadm -m discovery -t sendtargets -p 192.168.1.50:3260

# Login to iSCSI target
sudo iscsiadm -m node -T iqn.2024-01.com.example:storage.target01 -p 192.168.1.50:3260 -l

# Configure automatic login
sudo iscsiadm -m node -T iqn.2024-01.com.example:storage.target01 -p 192.168.1.50:3260 --op update -n node.startup -v automatic

# Configure CHAP authentication
sudo iscsiadm -m node -T iqn.2024-01.com.example:storage.target01 -p 192.168.1.50:3260 \
  --op update -n node.session.auth.authmethod -v CHAP
sudo iscsiadm -m node -T iqn.2024-01.com.example:storage.target01 -p 192.168.1.50:3260 \
  --op update -n node.session.auth.username -v myuser
sudo iscsiadm -m node -T iqn.2024-01.com.example:storage.target01 -p 192.168.1.50:3260 \
  --op update -n node.session.auth.password -v mypassword

# Performance tuning for iSCSI
cat >> /etc/iscsi/iscsid.conf << 'EOF'
# Performance settings
node.session.cmds_max = 256
node.session.queue_depth = 128
node.conn[0].iscsi.MaxRecvDataSegmentLength = 262144
node.conn[0].iscsi.MaxXmitDataSegmentLength = 262144
node.session.iscsi.FirstBurstLength = 262144
node.session.iscsi.MaxBurstLength = 16776192
node.conn[0].iscsi.HeaderDigest = None
node.conn[0].iscsi.DataDigest = None
EOF

# Create systemd mount unit for iSCSI device
cat > /etc/systemd/system/mnt-iscsi.mount << 'EOF'
[Unit]
Description=iSCSI Storage Mount
After=network-online.target iscsid.service

[Mount]
What=/dev/disk/by-path/ip-192.168.1.50:3260-iscsi-iqn.2024-01.com.example:storage.target01-lun-0
Where=/mnt/iscsi
Type=xfs
Options=defaults,_netdev

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable mnt-iscsi.mount
sudo systemctl start mnt-iscsi.mount

# Install FC utilities
sudo zypper install sg3_utils

# Scan for FC devices
sudo rescan-scsi-bus.sh
sudo lsscsi
sudo systool -c fc_host -v

# Configure FC parameters
cat > /etc/modprobe.d/qla2xxx.conf << 'EOF'
# QLogic FC HBA settings
options qla2xxx ql2xmaxqdepth=64 ql2xloginretrycount=30
EOF

cat > /etc/modprobe.d/lpfc.conf << 'EOF'
# Emulex FC HBA settings
options lpfc lpfc_lun_queue_depth=64 lpfc_hba_queue_depth=8192
EOF

# Persistent binding for FC devices
cat >> /etc/multipath/bindings << 'EOF'
# Persistent multipath bindings
mpatha 360060160844032007e8b3e5f8c53e411
mpathb 360060160844032007e8b3e5f8c53e412
mpathc 360060160844032007e8b3e5f8c53e413
EOF

# Zone configuration verification
sudo systool -c fc_remote_ports -v | grep port_name

# Create Btrfs filesystem with RAID
sudo mkfs.btrfs -L mydata -d raid1 -m raid1 /dev/sdb /dev/sdc

# Mount with compression
sudo mount -o compress=zstd:3,autodefrag /dev/sdb /mnt/btrfs

# Create subvolumes
sudo btrfs subvolume create /mnt/btrfs/home
sudo btrfs subvolume create /mnt/btrfs/data
sudo btrfs subvolume create /mnt/btrfs/snapshots

# Configure quotas
sudo btrfs quota enable /mnt/btrfs
sudo btrfs qgroup create 1/0 /mnt/btrfs
sudo btrfs qgroup limit 100G 1/0 /mnt/btrfs

# Create snapshots
sudo btrfs subvolume snapshot -r /mnt/btrfs/home /mnt/btrfs/snapshots/home-$(date +%Y%m%d)

# Schedule automatic snapshots
cat > /usr/local/bin/btrfs-snapshot.sh << 'EOF'
#!/bin/bash
# Automated Btrfs snapshots

SUBVOLUMES="home data"
SNAPSHOT_DIR="/mnt/btrfs/snapshots"
RETENTION_DAYS=7

for subvol in $SUBVOLUMES; do
    # Create snapshot
    sudo btrfs subvolume snapshot -r \
        /mnt/btrfs/$subvol \
        $SNAPSHOT_DIR/$subvol-$(date +%Y%m%d-%H%M%S)

    # Clean old snapshots
    find $SNAPSHOT_DIR -name "$subvol-*" -type d -mtime +$RETENTION_DAYS |
    while read snapshot; do
        sudo btrfs subvolume delete "$snapshot"
    done
done
EOF

chmod +x /usr/local/bin/btrfs-snapshot.sh

# Add to crontab
echo "0 */6 * * * /usr/local/bin/btrfs-snapshot.sh" | crontab -

# Check filesystem status
sudo btrfs filesystem show /mnt/btrfs
sudo btrfs filesystem df /mnt/btrfs
sudo btrfs device stats /mnt/btrfs

# Add device to existing filesystem
sudo btrfs device add /dev/sdd /mnt/btrfs
sudo btrfs balance start -dconvert=raid1 -mconvert=raid1 /mnt/btrfs

# Replace failed device
sudo btrfs replace start -f /dev/sdc /dev/sde /mnt/btrfs
sudo btrfs replace status /mnt/btrfs

# Scrub filesystem
sudo btrfs scrub start /mnt/btrfs
sudo btrfs scrub status /mnt/btrfs

# Balance filesystem
sudo btrfs balance start -dusage=50 -musage=70 /mnt/btrfs
sudo btrfs balance status /mnt/btrfs

# Send/Receive for backup
sudo btrfs send /mnt/btrfs/snapshots/home-20240215 | \
    sudo btrfs receive /backup/btrfs/

# Check current I/O scheduler
cat /sys/block/sda/queue/scheduler

# Configure I/O schedulers
cat > /etc/udev/rules.d/60-io-scheduler.rules << 'EOF'
# Set I/O scheduler based on device type
# NVMe devices - none
ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/scheduler}="none"
# SSD devices - mq-deadline
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="mq-deadline"
# HDD devices - bfq
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="bfq"
EOF

# Tune I/O scheduler parameters
# For mq-deadline
echo 100 > /sys/block/sda/queue/iosched/read_expire
echo 3000 > /sys/block/sda/queue/iosched/write_expire

# For bfq
echo 0 > /sys/block/sdb/queue/iosched/low_latency
echo 180000 > /sys/block/sdb/queue/iosched/timeout_sync

# Configure read-ahead
blockdev --setra 4096 /dev/sda

# Persistent read-ahead configuration
cat > /etc/udev/rules.d/61-readahead.rules << 'EOF'
# Set read-ahead for block devices
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{bdi/read_ahead_kb}="2048"
ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{bdi/read_ahead_kb}="1024"
EOF

# Configure queue depth
echo 256 > /sys/block/sda/queue/nr_requests

# Enable NCQ for SATA devices
echo 31 > /sys/block/sda/device/queue_depth

# Disable write barriers (use with caution)
mount -o remount,nobarrier /data

# Configure stripe parameters for RAID
echo 256 > /sys/block/md0/md/stripe_cache_size
echo 8192 > /sys/block/md0/queue/read_ahead_kb

# Configure dirty ratio for write caching
cat >> /etc/sysctl.d/storage-performance.conf << 'EOF'
# Storage performance tuning
vm.dirty_ratio = 5
vm.dirty_background_ratio = 2
vm.dirty_writeback_centisecs = 100
vm.dirty_expire_centisecs = 1000
vm.vfs_cache_pressure = 50
vm.swappiness = 10

# For database workloads
vm.zone_reclaim_mode = 0
vm.min_free_kbytes = 262144

# Transparent Huge Pages (disable for databases)
kernel.transparent_hugepage = madvise
EOF

sudo sysctl -p /etc/sysctl.d/storage-performance.conf

# Configure ARC size for ZFS (if using)
echo "options zfs zfs_arc_max=8589934592" > /etc/modprobe.d/zfs.conf

# Install monitoring tools
sudo zypper install sysstat iotop iostat dstat

# Configure sysstat collection
cat > /etc/sysconfig/sysstat << 'EOF'
# Sysstat configuration
HISTORY=31
COMPRESSAFTER=10
SADC_OPTIONS="-S DISK"
SA_DIR=/var/log/sa
ZIP="bzip2"
EOF

# Create storage monitoring script
cat > /usr/local/bin/storage-monitor.sh << 'EOF'
#!/bin/bash
# Storage monitoring script

LOG_DIR="/var/log/storage-monitor"
mkdir -p $LOG_DIR

# Collect I/O statistics
iostat -x 1 60 > $LOG_DIR/iostat-$(date +%Y%m%d-%H%M%S).log &

# Monitor specific devices
for device in sda sdb md0 dm-0; do
    if [ -b /dev/$device ]; then
        iostat -x /dev/$device 1 60 > $LOG_DIR/iostat-$device-$(date +%Y%m%d-%H%M%S).log &
    fi
done

# Collect Ceph statistics (if applicable)
if command -v ceph >/dev/null; then
    ceph -s > $LOG_DIR/ceph-status-$(date +%Y%m%d-%H%M%S).log
    ceph osd perf > $LOG_DIR/ceph-osd-perf-$(date +%Y%m%d-%H%M%S).log
    ceph pg dump > $LOG_DIR/ceph-pg-dump-$(date +%Y%m%d-%H%M%S).log
fi

# NFS statistics
if systemctl is-active nfs-server >/dev/null; then
    nfsstat -s > $LOG_DIR/nfsstat-$(date +%Y%m%d-%H%M%S).log
fi

# Generate summary report
cat > $LOG_DIR/summary-$(date +%Y%m%d-%H%M%S).txt << SUMMARY
Storage Performance Summary - $(date)
=====================================

Disk Usage:
$(df -h)

Active Mounts:
$(mount | grep -E "ext4|xfs|btrfs|nfs|ceph")

I/O Statistics:
$(iostat -x 1 1 | grep -E "Device|sd|md|dm")

Top I/O Processes:
$(iotop -b -n 1 | head -20)

SUMMARY

echo "Monitoring data collected in $LOG_DIR"
EOF

chmod +x /usr/local/bin/storage-monitor.sh

# Schedule regular monitoring
echo "*/5 * * * * /usr/local/bin/storage-monitor.sh" | crontab -

# Storage alert script
cat > /usr/local/bin/storage-alerts.sh << 'EOF'
#!/bin/bash
# Storage alert monitoring

EMAIL="storage-admin@example.com"
ALERT_LOG="/var/log/storage-alerts.log"

# Check disk usage
df -h | awk 'NR>1 && $5+0 > 85 {print "ALERT: " $6 " is " $5 " full"}' | \
while read alert; do
    echo "$(date): $alert" >> $ALERT_LOG
    echo "$alert" | mail -s "Storage Alert: Disk Usage" $EMAIL
done

# Check I/O wait
IOWAIT=$(iostat -c 1 2 | tail -1 | awk '{print $4}')
if (( $(echo "$IOWAIT > 50" | bc -l) )); then
    echo "$(date): High I/O wait: $IOWAIT%" >> $ALERT_LOG
    echo "I/O wait is $IOWAIT%" | mail -s "Storage Alert: High I/O Wait" $EMAIL
fi

# Check multipath status
if command -v multipath >/dev/null; then
    FAILED_PATHS=$(multipath -ll | grep -c "failed")
    if [ $FAILED_PATHS -gt 0 ]; then
        echo "$(date): $FAILED_PATHS failed multipath(s)" >> $ALERT_LOG
        multipath -ll | mail -s "Storage Alert: Failed Multipath" $EMAIL
    fi
fi

# Check Ceph health
if command -v ceph >/dev/null; then
    HEALTH=$(ceph health)
    if [[ "$HEALTH" != "HEALTH_OK" ]]; then
        echo "$(date): Ceph cluster unhealthy: $HEALTH" >> $ALERT_LOG
        ceph -s | mail -s "Storage Alert: Ceph Health" $EMAIL
    fi
fi
EOF

chmod +x /usr/local/bin/storage-alerts.sh

# Add to monitoring
echo "*/10 * * * * /usr/local/bin/storage-alerts.sh" | crontab -

# Backup script for various storage types
cat > /usr/local/bin/storage-backup.sh << 'EOF'
#!/bin/bash
# Comprehensive storage backup script

BACKUP_DIR="/backup"
LOG_FILE="/var/log/storage-backup.log"
DATE=$(date +%Y%m%d-%H%M%S)

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a $LOG_FILE
}

# Backup Ceph configuration
backup_ceph() {
    log "Starting Ceph backup"
    mkdir -p $BACKUP_DIR/ceph/$DATE

    # Backup Ceph configuration
    cp -r /etc/ceph $BACKUP_DIR/ceph/$DATE/

    # Export Ceph keyrings
    ceph auth export > $BACKUP_DIR/ceph/$DATE/ceph-auth.export

    # Backup CRUSH map
    ceph osd getcrushmap -o $BACKUP_DIR/ceph/$DATE/crushmap.bin

    # List current configuration
    ceph osd tree > $BACKUP_DIR/ceph/$DATE/osd-tree.txt
    ceph -s > $BACKUP_DIR/ceph/$DATE/cluster-status.txt

    log "Ceph backup completed"
}

# Backup NFS configuration
backup_nfs() {
    log "Starting NFS backup"
    mkdir -p $BACKUP_DIR/nfs/$DATE

    # Backup NFS configuration
    cp /etc/exports* $BACKUP_DIR/nfs/$DATE/
    cp /etc/idmapd.conf $BACKUP_DIR/nfs/$DATE/
    cp /etc/sysconfig/nfs $BACKUP_DIR/nfs/$DATE/

    # Backup Pacemaker configuration
    crm configure show > $BACKUP_DIR/nfs/$DATE/pacemaker-config.txt

    log "NFS backup completed"
}

# Backup multipath configuration
backup_multipath() {
    log "Starting multipath backup"
    mkdir -p $BACKUP_DIR/multipath/$DATE

    # Backup multipath configuration
    cp /etc/multipath.conf $BACKUP_DIR/multipath/$DATE/
    cp /etc/multipath/bindings $BACKUP_DIR/multipath/$DATE/

    # Current multipath status
    multipath -ll > $BACKUP_DIR/multipath/$DATE/multipath-status.txt

    log "Multipath backup completed"
}

# Backup Btrfs snapshots
backup_btrfs() {
    log "Starting Btrfs backup"
    mkdir -p $BACKUP_DIR/btrfs/$DATE

    # List all subvolumes and snapshots
    btrfs subvolume list /mnt/btrfs > $BACKUP_DIR/btrfs/$DATE/subvolumes.txt

    # Send snapshots to backup location
    for snapshot in /mnt/btrfs/snapshots/*; do
        if [ -d "$snapshot" ]; then
            snapshot_name=$(basename "$snapshot")
            log "Backing up snapshot: $snapshot_name"
            btrfs send "$snapshot" | \
                btrfs receive $BACKUP_DIR/btrfs/$DATE/
        fi
    done

    log "Btrfs backup completed"
}

# Main backup execution
log "Starting storage backup"

backup_ceph
backup_nfs
backup_multipath
backup_btrfs

# Compress backup
tar -czf $BACKUP_DIR/storage-backup-$DATE.tar.gz -C $BACKUP_DIR \
    ceph/$DATE nfs/$DATE multipath/$DATE btrfs/$DATE

# Clean up uncompressed files
rm -rf $BACKUP_DIR/{ceph,nfs,multipath,btrfs}/$DATE

log "Storage backup completed: $BACKUP_DIR/storage-backup-$DATE.tar.gz"

# Rotate old backups (keep last 7 days)
find $BACKUP_DIR -name "storage-backup-*.tar.gz" -mtime +7 -delete
EOF

chmod +x /usr/local/bin/storage-backup.sh

# Schedule backup
echo "0 2 * * * /usr/local/bin/storage-backup.sh" | crontab -

# Disaster recovery script
cat > /usr/local/bin/storage-disaster-recovery.sh << 'EOF'
#!/bin/bash
# Storage disaster recovery procedures

BACKUP_FILE=$1
RESTORE_DIR="/restore"

if [ -z "$BACKUP_FILE" ]; then
    echo "Usage: $0 <backup-file>"
    exit 1
fi

# Extract backup
mkdir -p $RESTORE_DIR
tar -xzf $BACKUP_FILE -C $RESTORE_DIR

# Restore Ceph configuration
restore_ceph() {
    echo "Restoring Ceph configuration..."

    # Stop Ceph services
    systemctl stop ceph.target

    # Restore configuration files
    cp -r $RESTORE_DIR/ceph/*/etc/ceph/* /etc/ceph/

    # Import auth keys
    ceph auth import < $RESTORE_DIR/ceph/*/ceph-auth.export

    # Restore CRUSH map
    crushtool -d $RESTORE_DIR/ceph/*/crushmap.bin -o /tmp/crushmap.txt
    # Review and apply: ceph osd setcrushmap -i /tmp/crushmap.txt

    echo "Ceph restore completed - manual verification required"
}

# Restore NFS configuration
restore_nfs() {
    echo "Restoring NFS configuration..."

    # Restore exports
    cp $RESTORE_DIR/nfs/*/exports* /etc/

    # Restore NFS configuration
    cp $RESTORE_DIR/nfs/*/idmapd.conf /etc/
    cp $RESTORE_DIR/nfs/*/nfs /etc/sysconfig/

    # Reload NFS exports
    exportfs -ra

    echo "NFS restore completed"
}

# Interactive menu
echo "Storage Disaster Recovery"
echo "========================="
echo "1. Restore Ceph configuration"
echo "2. Restore NFS configuration"
echo "3. Restore multipath configuration"
echo "4. Restore all"
echo "5. Exit"

read -p "Select option: " option

case $option in
    1) restore_ceph ;;
    2) restore_nfs ;;
    3) echo "Restore multipath manually from $RESTORE_DIR/multipath" ;;
    4) restore_ceph; restore_nfs ;;
    5) exit 0 ;;
    *) echo "Invalid option" ;;
esac
EOF

chmod +x /usr/local/bin/storage-disaster-recovery.sh

# Ceph encryption at rest
# Enable encryption for new OSDs
ceph config set osd osd_dmcrypt_type luks

# Configure encrypted RBD images
rbd create --size 10G --pool rbd-pool \
    --image-feature layering,exclusive-lock,object-map,fast-diff \
    encrypted-disk

# Set up LUKS encryption for RBD
rbd map rbd-pool/encrypted-disk
cryptsetup luksFormat /dev/rbd0
cryptsetup luksOpen /dev/rbd0 encrypted-rbd
mkfs.xfs /dev/mapper/encrypted-rbd

# NFS Kerberos authentication
cat > /etc/krb5.conf << 'EOF'
[libdefaults]
    default_realm = EXAMPLE.COM
    dns_lookup_realm = false
    dns_lookup_kdc = false

[realms]
    EXAMPLE.COM = {
        kdc = kdc.example.com
        admin_server = kdc.example.com
    }

[domain_realm]
    .example.com = EXAMPLE.COM
    example.com = EXAMPLE.COM
EOF

# Configure NFS for Kerberos
sed -i 's/RPCGSSDARGS=""/RPCGSSDARGS="-n"/' /etc/sysconfig/nfs
sed -i 's/RPCSVCGSSDARGS=""/RPCSVCGSSDARGS="-n"/' /etc/sysconfig/nfs

# Update exports for Kerberos
cat >> /etc/exports << 'EOF'
/srv/nfs/secure *(rw,sync,sec=krb5p,no_root_squash)
EOF

# Start required services
systemctl enable rpc-gssd
systemctl start rpc-gssd
systemctl restart nfs-server

# Create storage best practices document
cat > /usr/local/share/storage-best-practices.md << 'EOF'
# SUSE Storage Best Practices

## Ceph Deployment
- Use dedicated networks for public and cluster traffic
- Minimum 3 MON nodes for quorum
- Use SSDs for journal/metadata
- Configure proper CRUSH rules for failure domains
- Regular scrubbing and deep-scrubbing
- Monitor cluster health continuously

## NFS Guidelines
- Use dedicated storage network
- Implement HA with DRBD/Pacemaker
- Configure appropriate export options
- Use NFSv4.2 for better performance
- Implement Kerberos for security
- Regular backup of export configuration

## SAN Best Practices
- Use multipath for redundancy
- Configure proper queue depths
- Implement zoning at switch level
- Regular firmware updates
- Monitor path status
- Document LUN mappings

## Performance Optimization
- Choose appropriate I/O schedulers
- Tune kernel parameters
- Use appropriate RAID levels
- Implement caching strategies
- Monitor I/O patterns
- Regular performance testing

## Security Measures
- Encrypt data at rest
- Use secure protocols
- Implement access controls
- Regular security audits
- Monitor access logs
- Keep systems updated

## Backup and Recovery
- Regular configuration backups
- Test recovery procedures
- Document recovery steps
- Maintain offsite copies
- Monitor backup success
- Regular DR drills
EOF