Skip to main content

Archive & Backup

Tar Operations​

Basic Tar Commands​

# Create archives
tar -cvf archive.tar files         # Create tar archive
tar -czf archive.tar.gz files      # Create compressed (gzip) archive
tar -cjf archive.tar.bz2 files     # Create bzip2 compressed archive
tar -cJf archive.tar.xz files      # Create xz compressed archive

# Extract archives
tar -xvf archive.tar               # Extract tar archive
tar -xzf archive.tar.gz            # Extract gzip archive
tar -xjf archive.tar.bz2           # Extract bzip2 archive
tar -xJf archive.tar.xz            # Extract xz archive

# List archive contents
tar -tvf archive.tar               # List files in archive
tar -tzf archive.tar.gz            # List files in compressed archive

Advanced Tar Options​

# Selective extraction
tar -xzf archive.tar.gz file1 file2  # Extract specific files
tar -xzf archive.tar.gz --wildcards "*.txt"  # Extract by pattern
tar -xzf archive.tar.gz -C /target/dir       # Extract to specific directory

# Incremental backups
tar -czf backup.tar.gz -g snapshot.snar /home  # Create incremental backup
tar -czf backup-inc.tar.gz -g snapshot.snar /home  # Create next increment

# Archive with exclusions
tar -czf backup.tar.gz --exclude="*.tmp" /home     # Exclude pattern
tar -czf backup.tar.gz --exclude-from=exclude.txt /home  # Exclude from file
tar -czf backup.tar.gz --exclude-vcs /home         # Exclude version control

# Preserve permissions and ownership
tar -czpf archive.tar.gz /home      # Preserve permissions
tar -czpf archive.tar.gz --same-owner /home  # Preserve ownership

Tar with Network​

# Create archive over network
tar -czf - /home | ssh user@remote "cat > backup.tar.gz"
ssh user@remote "tar -czf - /remote/path" | tar -xzf -

# Pipe to remote extraction
tar -czf - /local/path | ssh user@remote "tar -xzf - -C /remote/path"

Gzip and Compression​

Gzip Operations​

# Compress files
gzip file.txt                      # Compress file (creates file.txt.gz)
gzip -9 file.txt                   # Maximum compression
gzip -1 file.txt                   # Fastest compression
gzip -r directory/                 # Compress all files in directory

# Decompress files
gunzip file.txt.gz                 # Decompress file
gzip -d file.txt.gz                # Alternative decompression
zcat file.txt.gz                   # View compressed file without extracting

# Multiple files
gzip file1.txt file2.txt           # Compress multiple files
gunzip *.gz                        # Decompress all gzip files

Other Compression Tools​

# Bzip2 (better compression)
bzip2 file.txt                     # Compress with bzip2
bunzip2 file.txt.bz2               # Decompress bzip2
bzcat file.txt.bz2                 # View bzip2 file

# XZ (best compression)
xz file.txt                        # Compress with xz
unxz file.txt.xz                   # Decompress xz
xzcat file.txt.xz                  # View xz file

# Parallel compression
pigz file.txt                      # Parallel gzip
pbzip2 file.txt                    # Parallel bzip2
pxz file.txt                       # Parallel xz

Zip Operations​

Basic Zip Commands​

# Create zip archives
zip archive.zip file1 file2        # Create zip archive
zip -r archive.zip directory/      # Recursively zip directory
zip -9 archive.zip files           # Maximum compression

# Extract zip archives
unzip archive.zip                  # Extract all files
unzip archive.zip -d /target/dir   # Extract to specific directory
unzip archive.zip file1.txt       # Extract specific file

# List and test
unzip -l archive.zip               # List files in archive
unzip -t archive.zip               # Test archive integrity

Advanced Zip Options​

# Password protection
zip -e secure.zip file.txt         # Create encrypted zip
unzip -P password secure.zip       # Extract with password

# Exclude patterns
zip -r archive.zip /home -x "*.tmp" "*.log"  # Exclude patterns
zip -r archive.zip /home -x@exclude.txt      # Exclude from file

# Update and refresh
zip -u archive.zip newfile.txt     # Update archive with new file
zip -f archive.zip                 # Freshen existing files only

Rsync - Remote Synchronization​

Basic Rsync Operations​

# Local synchronization
rsync -av source/ destination/     # Archive mode (preserve attrs)
rsync -avz source/ destination/    # With compression
rsync -avh source/ destination/    # Human-readable sizes

# Remote synchronization
rsync -avz /local/path/ user@remote:/remote/path/
rsync -avz user@remote:/remote/path/ /local/path/

Advanced Rsync Options​

# Incremental backups
rsync -avz --delete source/ backup/         # Mirror (delete extra files)
rsync -avz --backup source/ backup/         # Keep backup of changed files
rsync -avz --backup-dir=../old source/ backup/  # Backup to specific dir

# Exclude patterns
rsync -avz --exclude="*.tmp" source/ backup/
rsync -avz --exclude-from=exclude.txt source/ backup/
rsync -avz --exclude=".*" source/ backup/   # Exclude hidden files

# Bandwidth and transfer control
rsync -avz --bwlimit=100 source/ backup/    # Limit bandwidth (KB/s)
rsync -avz --partial source/ backup/        # Keep partial transfers
rsync -avz --progress source/ backup/       # Show progress

Rsync for Backups​

# Daily backup script
#!/bin/bash
DATE=$(date +%Y%m%d)
rsync -avz --delete --backup --backup-dir="../backup-$DATE" \
  /home/user/ /backup/current/

# Remote backup with SSH key
rsync -avz -e "ssh -i ~/.ssh/backup_key" \
  /home/user/ backup@remote:/backups/$(hostname)/

# Exclude system files
rsync -avz --exclude-from=- /home/user/ /backup/user/ << 'EOF'
.cache/
.tmp/
Downloads/
*.iso
*.log
EOF

DD - Low-Level Backup​

Basic DD Operations​

# Create disk images
dd if=/dev/sda of=disk_image.img   # Create full disk image
dd if=/dev/sda1 of=partition.img   # Create partition image
dd if=/dev/sda of=disk.img bs=4M   # Use 4MB block size (faster)

# Restore from images
dd if=disk_image.img of=/dev/sda   # Restore disk image
dd if=partition.img of=/dev/sda1   # Restore partition

# Monitor progress
dd if=/dev/sda of=disk.img bs=4M status=progress  # Show progress

Advanced DD Usage​

# Backup MBR
dd if=/dev/sda of=mbr_backup.img bs=512 count=1

# Create bootable USB
dd if=linux.iso of=/dev/sdb bs=4M status=progress

# Secure wipe
dd if=/dev/zero of=/dev/sda bs=4M    # Zero out disk
dd if=/dev/urandom of=/dev/sda bs=4M # Random data wipe

# Network backup
dd if=/dev/sda bs=4M | gzip | ssh user@remote "cat > disk_backup.img.gz"

Backup Strategies​

Full Backup Strategy​

#!/bin/bash
# Full backup script
BACKUP_DIR="/backups"
SOURCE_DIRS="/home /etc /var/log /usr/local"
DATE=$(date +%Y%m%d-%H%M%S)

for dir in $SOURCE_DIRS; do
    echo "Backing up $dir..."
    tar -czf "$BACKUP_DIR/$(basename $dir)-full-$DATE.tar.gz" "$dir"
done

Incremental Backup Strategy​

#!/bin/bash
# Incremental backup with tar
BACKUP_DIR="/backups"
SOURCE="/home"
SNAPSHOT="$BACKUP_DIR/snapshot.snar"
DATE=$(date +%Y%m%d)

# Create incremental backup
tar -czf "$BACKUP_DIR/home-inc-$DATE.tar.gz" \
    -g "$SNAPSHOT" "$SOURCE"

# Restore incremental backups (in order)
tar -xzf home-full-20231201.tar.gz -g /dev/null
tar -xzf home-inc-20231202.tar.gz -g /dev/null
tar -xzf home-inc-20231203.tar.gz -g /dev/null

Differential Backup Strategy​

#!/bin/bash
# Differential backup using rsync
BACKUP_DIR="/backups"
SOURCE="/home"
FULL_BACKUP="$BACKUP_DIR/full"
DIFF_BACKUP="$BACKUP_DIR/diff-$(date +%Y%m%d)"

# Create full backup (weekly)
if [ $(date +%w) -eq 0 ]; then
    rsync -av --delete "$SOURCE/" "$FULL_BACKUP/"
fi

# Create differential backup (daily)
rsync -av --compare-dest="$FULL_BACKUP" "$SOURCE/" "$DIFF_BACKUP/"

Backup Tools​

Bacula - Professional Backup Solution​

# Install Bacula
apt-get install bacula-server bacula-client bacula-console

# Basic Bacula configuration
# /etc/bacula/bacula-dir.conf
Job {
  Name = "BackupClient1"
  Type = Backup
  Client = client1-fd
  FileSet = "Full Set"
  Schedule = "WeeklyCycle"
  Storage = File
  Messages = Standard
  Pool = Default
  Write Bootstrap = "/var/lib/bacula/%c.bsr"
}

# Run backup
bconsole
run job=BackupClient1

Duplicity - Encrypted Incremental Backups​

# Install Duplicity
apt-get install duplicity

# Create encrypted backup
duplicity /home file:///backup/duplicity

# Incremental backup
duplicity /home file:///backup/duplicity

# Restore full backup
duplicity restore file:///backup/duplicity /restore/path

# Restore specific file
duplicity restore --file-to-restore home/user/file.txt \
    file:///backup/duplicity /restore/path

Borgbackup - Deduplicating Backups​

# Install Borgbackup
pip install borgbackup

# Initialize repository
borg init --encryption=repokey /path/to/repo

# Create backup
borg create /path/to/repo::backup-$(date +%Y%m%d) /home

# List backups
borg list /path/to/repo

# Mount backup
borg mount /path/to/repo::backup-20231201 /mnt/backup

Remote Backup Solutions​

AWS S3 Backup​

# Install AWS CLI
pip install awscli

# Configure AWS credentials
aws configure

# Sync to S3
aws s3 sync /home/user/ s3://my-backup-bucket/user/

# Restore from S3
aws s3 sync s3://my-backup-bucket/user/ /restore/path/

# Lifecycle policy for old backups
aws s3api put-bucket-lifecycle-configuration \
    --bucket my-backup-bucket \
    --lifecycle-configuration file://lifecycle.json

Rclone - Cloud Storage Sync​

# Install rclone
curl https://rclone.org/install.sh | sudo bash

# Configure remote
rclone config

# Sync to cloud
rclone sync /home/user/ remote:backup/user/

# Backup with encryption
rclone sync /home/user/ remote:backup/user/ --crypt-password

# Mount cloud storage
rclone mount remote:backup/ /mnt/cloud/ --daemon

SSH/SCP Backup Scripts​

#!/bin/bash
# SSH backup script
REMOTE_HOST="backup.example.com"
REMOTE_USER="backup"
REMOTE_PATH="/backups/$(hostname)"
LOCAL_PATH="/home"

# Create backup archive
tar -czf /tmp/backup-$(date +%Y%m%d).tar.gz "$LOCAL_PATH"

# Transfer to remote
scp /tmp/backup-$(date +%Y%m%d).tar.gz \
    "$REMOTE_USER@$REMOTE_HOST:$REMOTE_PATH/"

# Clean local temporary file
rm /tmp/backup-$(date +%Y%m%d).tar.gz

Backup Automation​

Cron Job Automation​

# Edit crontab
crontab -e

# Daily backup at 2 AM
0 2 * * * /usr/local/bin/backup.sh

# Weekly full backup on Sunday
0 1 * * 0 /usr/local/bin/full-backup.sh

# Monthly cleanup
0 3 1 * * /usr/local/bin/cleanup-old-backups.sh

Systemd Timer Automation​

# Create backup.timer
cat > /etc/systemd/system/backup.timer << 'EOF'
[Unit]
Description=Daily backup timer
Requires=backup.service

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
EOF

# Create backup.service
cat > /etc/systemd/system/backup.service << 'EOF'
[Unit]
Description=Backup service
After=network.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/backup.sh
User=backup
Group=backup
EOF

# Enable and start
systemctl daemon-reload
systemctl enable backup.timer
systemctl start backup.timer

Backup Script with Logging​

#!/bin/bash
# Comprehensive backup script with logging
LOGFILE="/var/log/backup.log"
BACKUP_DIR="/backups"
SOURCE_DIRS="/home /etc /var/log"
DATE=$(date +%Y%m%d-%H%M%S)
RETENTION_DAYS=30

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOGFILE"
}

# Start backup
log "Starting backup process"

# Create backups
for dir in $SOURCE_DIRS; do
    backup_file="$BACKUP_DIR/$(basename $dir)-$DATE.tar.gz"
    log "Backing up $dir to $backup_file"

    if tar -czf "$backup_file" "$dir" 2>>"$LOGFILE"; then
        log "Successfully backed up $dir"
    else
        log "ERROR: Failed to backup $dir"
    fi
done

# Clean old backups
log "Cleaning backups older than $RETENTION_DAYS days"
find "$BACKUP_DIR" -name "*.tar.gz" -mtime +$RETENTION_DAYS -delete

log "Backup process completed"

Backup Verification​

Verification Scripts​

#!/bin/bash
# Backup verification script
BACKUP_DIR="/backups"
VERIFICATION_LOG="/var/log/backup-verification.log"

verify_backup() {
    local backup_file="$1"
    echo "Verifying $backup_file..." | tee -a "$VERIFICATION_LOG"

    # Test archive integrity
    if tar -tzf "$backup_file" > /dev/null 2>&1; then
        echo "✓ Archive integrity: PASS" | tee -a "$VERIFICATION_LOG"
    else
        echo "✗ Archive integrity: FAIL" | tee -a "$VERIFICATION_LOG"
        return 1
    fi

    # Check file count
    file_count=$(tar -tzf "$backup_file" | wc -l)
    echo "  File count: $file_count" | tee -a "$VERIFICATION_LOG"

    # Check archive size
    size=$(ls -lh "$backup_file" | awk '{print $5}')
    echo "  Archive size: $size" | tee -a "$VERIFICATION_LOG"

    return 0
}

# Verify all backups
for backup in "$BACKUP_DIR"/*.tar.gz; do
    verify_backup "$backup"
done

Checksums and Integrity​

# Create checksums
find /backups -name "*.tar.gz" -exec sha256sum {} \; > checksums.sha256

# Verify checksums
sha256sum -c checksums.sha256

# MD5 checksums (faster, less secure)
find /backups -name "*.tar.gz" -exec md5sum {} \; > checksums.md5
md5sum -c checksums.md5

# Verify tar archive integrity
tar -tzf backup.tar.gz > /dev/null && echo "Archive OK" || echo "Archive CORRUPTED"

Database Backup Strategies​

MySQL/MariaDB Backup​

# Full database backup
mysqldump -u root -p --all-databases > full_backup.sql
mysqldump -u root -p --single-transaction --routines --triggers \
    database_name > database_backup.sql

# Incremental backup (using binary logs)
mysqldump -u root -p --master-data=2 --single-transaction \
    database_name > database_backup.sql

# Point-in-time recovery
mysql -u root -p database_name < database_backup.sql
mysqlbinlog --start-datetime="2023-12-01 10:00:00" \
    --stop-datetime="2023-12-01 10:30:00" \
    /var/log/mysql/mysql-bin.000001 | mysql -u root -p database_name

# Automated backup script
#!/bin/bash
BACKUP_DIR="/backups/mysql"
DATE=$(date +%Y%m%d)
mysqldump -u backup_user -p$BACKUP_PASSWORD \
    --single-transaction --routines --triggers \
    --all-databases | gzip > "$BACKUP_DIR/mysql-$DATE.sql.gz"

PostgreSQL Backup​

# Full database backup
pg_dump -U postgres database_name > database_backup.sql
pg_dumpall -U postgres > full_backup.sql

# Compressed backup
pg_dump -U postgres -Fc database_name > database_backup.dump

# Directory format backup
pg_dump -U postgres -Fd database_name -f backup_directory

# Restore database
psql -U postgres -d database_name < database_backup.sql
pg_restore -U postgres -d database_name database_backup.dump

# Continuous archiving
# postgresql.conf
archive_mode = on
archive_command = 'cp %p /backup/archive/%f'
wal_level = replica

MongoDB Backup​

# Database backup
mongodump --db database_name --out /backup/mongodb/

# All databases
mongodump --out /backup/mongodb/

# Specific collection
mongodump --db database_name --collection collection_name --out /backup/

# Restore database
mongorestore --db database_name /backup/mongodb/database_name/

# Backup with authentication
mongodump --username user --password pass --authenticationDatabase admin \
    --db database_name --out /backup/

Disaster Recovery​

System Recovery Planning​

# Create system recovery documentation
cat > /backup/recovery-plan.md << 'EOF'
# Disaster Recovery Plan

## Critical Systems
- Database server: 192.168.1.10
- Web server: 192.168.1.11
- File server: 192.168.1.12

## Recovery Priority
1. Database server (RTO: 1 hour)
2. Web server (RTO: 2 hours)
3. File server (RTO: 4 hours)

## Backup Locations
- Local: /backups/
- Remote: backup.example.com:/backups/
- Cloud: s3://disaster-recovery-bucket/

## Recovery Procedures
1. Restore from latest backup
2. Verify database integrity
3. Test application functionality
4. Update DNS if needed
EOF

Bare Metal Recovery​

# Create system image
dd if=/dev/sda of=/backup/system-image.img bs=4M status=progress

# Network boot recovery
# Setup PXE server with recovery image
dnsmasq --enable-tftp --tftp-root=/tftpboot --dhcp-range=192.168.1.100,192.168.1.150

# Restore system image
dd if=/backup/system-image.img of=/dev/sda bs=4M status=progress

Cloud Disaster Recovery​

# AWS EC2 snapshot automation
#!/bin/bash
INSTANCE_ID="i-1234567890abcdef0"
DESCRIPTION="Automated snapshot $(date +%Y%m%d)"

# Create snapshot
aws ec2 create-snapshot \
    --volume-id $(aws ec2 describe-instances \
        --instance-ids $INSTANCE_ID \
        --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' \
        --output text) \
    --description "$DESCRIPTION"

# Delete old snapshots (keep 7 days)
aws ec2 describe-snapshots --owner-ids self \
    --query 'Snapshots[?StartTime<=`'$(date -d '7 days ago' --iso-8601)'`].SnapshotId' \
    --output text | xargs -n1 aws ec2 delete-snapshot --snapshot-id

Best Practices​

Backup Strategy Guidelines​

  1. 3-2-1 Rule: 3 copies, 2 different media types, 1 offsite
  2. Regular Testing: Test restores monthly
  3. Automation: Automate backup processes
  4. Monitoring: Monitor backup success/failure
  5. Documentation: Document recovery procedures
  6. Encryption: Encrypt sensitive backups
  7. Retention: Define retention policies

Backup Security​

# Encrypt backups
gpg --symmetric --cipher-algo AES256 backup.tar.gz

# Decrypt backup
gpg --decrypt backup.tar.gz.gpg > backup.tar.gz

# Secure backup storage
chmod 600 /backups/*
chown backup:backup /backups/*

# Backup over SSH with key authentication
ssh-keygen -t rsa -b 4096 -f ~/.ssh/backup_key
ssh-copy-id -i ~/.ssh/backup_key.pub backup@remote-server

Monitoring and Alerting​

# Backup monitoring script
#!/bin/bash
BACKUP_DIR="/backups"
MAX_AGE=25  # hours
ALERT_EMAIL="admin@example.com"

check_backup_age() {
    local backup_file="$1"
    local age=$(find "$backup_file" -mtime +1 -print)

    if [[ -n "$age" ]]; then
        echo "WARNING: Backup $backup_file is older than $MAX_AGE hours" | \
            mail -s "Backup Alert" "$ALERT_EMAIL"
    fi
}

# Check all backups
for backup in "$BACKUP_DIR"/*.tar.gz; do
    check_backup_age "$backup"
done

Recovery Testing​

#!/bin/bash
# Automated recovery testing
TEST_DIR="/tmp/recovery-test"
BACKUP_FILE="/backups/latest-backup.tar.gz"

# Create test environment
mkdir -p "$TEST_DIR"
cd "$TEST_DIR"

# Extract backup
tar -xzf "$BACKUP_FILE"

# Run verification tests
echo "Testing file integrity..."
find . -type f -exec file {} \; | grep -v "ASCII\|UTF-8\|binary" | \
    head -10

echo "Testing database backup..."
if [[ -f database_backup.sql ]]; then
    mysql -u test_user -p test_db < database_backup.sql
    mysql -u test_user -p -e "SELECT COUNT(*) FROM information_schema.tables;"
fi

# Cleanup
cd /
rm -rf "$TEST_DIR"