Advanced Linux Systems Administration: Enterprise Guide 2025

Introduction

Enterprise Linux systems administration has evolved significantly in 2025, requiring deep understanding of modern kernel features, container technologies, security frameworks, and automation patterns. This comprehensive guide covers advanced topics essential for managing enterprise-grade Linux infrastructure.

Chapter 1: Advanced Filesystem Management

Enterprise Bind Mount Strategies

Bind mounts are crucial for enterprise container orchestration, chroot environments, and multi-tenant systems.

#!/bin/bash
# Enterprise bind mount management script

class BindMountManager {
    private:
        std::vector<std::string> active_mounts;
        std::string audit_log_path;
        
    public:
        bool create_secure_bind_mount(const std::string& source, 
                                    const std::string& target,
                                    const MountOptions& options) {
            // Validate source exists and is accessible
            if (!filesystem_security_validator.validate_path(source)) {
                audit_logger.log_security_violation("Invalid bind mount source", source);
                return false;
            }
            
            // Create target directory with proper permissions
            std::filesystem::create_directories(target);
            std::filesystem::permissions(target, options.permissions);
            
            // Execute bind mount with security flags
            std::string mount_cmd = format_mount_command(source, target, options);
            int result = execute_with_capabilities(mount_cmd, CAP_SYS_ADMIN);
            
            if (result == 0) {
                active_mounts.push_back(target);
                audit_logger.log_mount_event("bind_mount_created", source, target);
                return true;
            }
            
            return false;
        }
        
        bool unmount_all_in_namespace(const std::string& namespace_id) {
            for (const auto& mount : active_mounts) {
                if (is_in_namespace(mount, namespace_id)) {
                    lazy_unmount(mount);
                }
            }
            return true;
        }
};

# Production bind mount with security hardening
create_enterprise_bind_mount() {
    local source="$1"
    local target="$2"
    local options="$3"
    
    # Validate inputs
    [[ -d "$source" ]] || { log_error "Source directory does not exist: $source"; return 1; }
    
    # Create target with restricted permissions
    mkdir -p "$target"
    chmod 750 "$target"
    
    # Apply SELinux context if enabled
    if selinux_enabled; then
        chcon --reference="$source" "$target"
    fi
    
    # Create bind mount with security options
    mount --bind \
          --make-private \
          -o nosuid,nodev,noexec \
          "$source" "$target"
    
    # Verify mount was successful
    if mountpoint -q "$target"; then
        log_info "Bind mount created: $source -> $target"
        echo "$target" >> /var/log/enterprise-mounts.log
        return 0
    else
        log_error "Failed to create bind mount: $source -> $target"
        return 1
    fi
}

# Enterprise mount monitoring
monitor_bind_mounts() {
    while IFS= read -r mount_point; do
        if ! mountpoint -q "$mount_point"; then
            alert_manager.send_alert("bind_mount_failure", {
                "mount_point": "$mount_point",
                "timestamp": "$(date -Iseconds)",
                "severity": "critical"
            })
        fi
    done < /var/log/enterprise-mounts.log
}

Advanced Filesystem Features

// Enterprise filesystem management in Go
package filesystem

import (
    "context"
    "fmt"
    "os"
    "path/filepath"
    "syscall"
    "time"
    
    "golang.org/x/sys/unix"
)

type EnterpriseFilesystemManager struct {
    auditLogger    *AuditLogger
    metricsClient  *MetricsClient
    quotaManager   *QuotaManager
    snapshotMgr    *SnapshotManager
}

// Advanced XFS quota management
func (efm *EnterpriseFilesystemManager) ConfigureXFSQuotas(
    mountPoint string, 
    userQuotas map[uint32]QuotaLimits,
    groupQuotas map[uint32]QuotaLimits) error {
    
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()
    
    // Enable quota accounting
    if err := efm.enableQuotaAccounting(mountPoint); err != nil {
        return fmt.Errorf("failed to enable quota accounting: %w", err)
    }
    
    // Configure user quotas
    for uid, limits := range userQuotas {
        quota := unix.Dqblk{
            Bhardlimit: limits.BlockHardLimit,
            Bsoftlimit: limits.BlockSoftLimit,
            Ihardlimit: limits.InodeHardLimit,
            Isoftlimit: limits.InodeSoftLimit,
            Btime:      uint64(limits.BlockGracePeriod.Unix()),
            Itime:      uint64(limits.InodeGracePeriod.Unix()),
        }
        
        if err := unix.Quotactl(unix.QCMD(unix.Q_SETQUOTA, unix.USRQUOTA),
                               mountPoint, int(uid), (*byte)(&quota)); err != nil {
            efm.auditLogger.LogQuotaError("user_quota_set_failed", uid, err)
            return err
        }
        
        efm.metricsClient.RecordQuotaConfiguration("user", uid, limits)
    }
    
    // Configure group quotas
    for gid, limits := range groupQuotas {
        quota := unix.Dqblk{
            Bhardlimit: limits.BlockHardLimit,
            Bsoftlimit: limits.BlockSoftLimit,
            Ihardlimit: limits.InodeHardLimit,
            Isoftlimit: limits.InodeSoftLimit,
        }
        
        if err := unix.Quotactl(unix.QCMD(unix.Q_SETQUOTA, unix.GRPQUOTA),
                               mountPoint, int(gid), (*byte)(&quota)); err != nil {
            return err
        }
    }
    
    efm.auditLogger.LogQuotaConfiguration(mountPoint, userQuotas, groupQuotas)
    return nil
}

// Enterprise snapshot management with CoW filesystems
func (efm *EnterpriseFilesystemManager) CreateEnterpriseSnapshot(
    sourcePath string, 
    snapshotName string,
    metadata SnapshotMetadata) (*Snapshot, error) {
    
    // Detect filesystem type
    fsType, err := efm.detectFilesystemType(sourcePath)
    if err != nil {
        return nil, err
    }
    
    var snapshot *Snapshot
    
    switch fsType {
    case "btrfs":
        snapshot, err = efm.createBtrfsSnapshot(sourcePath, snapshotName, metadata)
    case "zfs":
        snapshot, err = efm.createZfsSnapshot(sourcePath, snapshotName, metadata)
    case "xfs":
        snapshot, err = efm.createXfsReflink(sourcePath, snapshotName, metadata)
    default:
        return nil, fmt.Errorf("unsupported filesystem for snapshots: %s", fsType)
    }
    
    if err != nil {
        efm.auditLogger.LogSnapshotError("snapshot_creation_failed", sourcePath, err)
        return nil, err
    }
    
    // Register snapshot for monitoring
    efm.snapshotMgr.RegisterSnapshot(snapshot)
    efm.metricsClient.RecordSnapshotCreated(snapshot.Name, snapshot.Size)
    
    return snapshot, nil
}

type QuotaLimits struct {
    BlockHardLimit    uint64
    BlockSoftLimit    uint64
    InodeHardLimit    uint64
    InodeSoftLimit    uint64
    BlockGracePeriod  time.Duration
    InodeGracePeriod  time.Duration
}

type SnapshotMetadata struct {
    Purpose     string
    Retention   time.Duration
    Tags        map[string]string
    Encrypted   bool
    Compressed  bool
}

Chapter 2: Advanced Namespace Management

Enterprise Container Namespaces

// Advanced namespace management in C
#define _GNU_SOURCE
#include <sched.h>
#include <sys/wait.h>
#include <sys/mount.h>
#include <sys/syscall.h>
#include <linux/sched.h>

struct enterprise_namespace_config {
    unsigned long clone_flags;
    char *hostname;
    char *domainname;
    uid_t uid_map_start;
    uid_t uid_map_length;
    gid_t gid_map_start;
    gid_t gid_map_length;
    char *root_dir;
    char **bind_mounts;
    size_t bind_mount_count;
    int security_level;
};

// Create enterprise namespace with security hardening
int create_enterprise_namespace(struct enterprise_namespace_config *config) {
    pid_t child_pid;
    int pipe_fd[2];
    char map_buf[256];
    
    // Create communication pipe
    if (pipe(pipe_fd) == -1) {
        perror("pipe");
        return -1;
    }
    
    // Configure clone flags for maximum isolation
    unsigned long clone_flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS |
                               CLONE_NEWUTS | CLONE_NEWIPC | CLONE_NEWNET |
                               CLONE_NEWCGROUP;
    
    if (config->security_level >= SECURITY_LEVEL_HIGH) {
        clone_flags |= CLONE_NEWTIME;  // Time namespace isolation
    }
    
    child_pid = clone(enterprise_namespace_child, 
                      malloc(4096) + 4096,  // Stack for child
                      clone_flags | SIGCHLD,
                      config);
    
    if (child_pid == -1) {
        perror("clone");
        return -1;
    }
    
    // Configure user namespace mappings
    configure_user_namespace_mappings(child_pid, config);
    
    // Signal child to continue
    close(pipe_fd[1]);
    if (write(pipe_fd[1], "continue", 8) != 8) {
        perror("write to pipe");
        return -1;
    }
    
    return child_pid;
}

// Child process in new namespace
int enterprise_namespace_child(void *arg) {
    struct enterprise_namespace_config *config = arg;
    char buf[64];
    int pipe_fd[2] = {3, 4};  // Assuming inherited pipe
    
    // Wait for parent to configure user namespace
    if (read(pipe_fd[0], buf, sizeof(buf)) == -1) {
        perror("read from pipe");
        return -1;
    }
    close(pipe_fd[0]);
    
    // Configure hostname and domain
    if (sethostname(config->hostname, strlen(config->hostname)) == -1) {
        perror("sethostname");
        return -1;
    }
    
    if (setdomainname(config->domainname, strlen(config->domainname)) == -1) {
        perror("setdomainname");
        return -1;
    }
    
    // Setup new root filesystem
    if (setup_enterprise_rootfs(config) == -1) {
        return -1;
    }
    
    // Apply security restrictions
    apply_namespace_security_restrictions(config);
    
    // Execute target process
    char *const argv[] = {"/bin/bash", NULL};
    char *const envp[] = {NULL};
    
    execve("/bin/bash", argv, envp);
    perror("execve");
    return -1;
}

// Configure user namespace ID mappings
int configure_user_namespace_mappings(pid_t child_pid, 
                                     struct enterprise_namespace_config *config) {
    char map_path[256];
    char map_content[256];
    int fd;
    
    // Configure UID mapping
    snprintf(map_path, sizeof(map_path), "/proc/%d/uid_map", child_pid);
    snprintf(map_content, sizeof(map_content), "%d %d %d\n",
             0, config->uid_map_start, config->uid_map_length);
    
    fd = open(map_path, O_WRONLY);
    if (fd == -1) {
        perror("open uid_map");
        return -1;
    }
    
    if (write(fd, map_content, strlen(map_content)) == -1) {
        perror("write uid_map");
        close(fd);
        return -1;
    }
    close(fd);
    
    // Deny setgroups for security
    snprintf(map_path, sizeof(map_path), "/proc/%d/setgroups", child_pid);
    fd = open(map_path, O_WRONLY);
    if (fd != -1) {
        write(fd, "deny", 4);
        close(fd);
    }
    
    // Configure GID mapping
    snprintf(map_path, sizeof(map_path), "/proc/%d/gid_map", child_pid);
    snprintf(map_content, sizeof(map_content), "%d %d %d\n",
             0, config->gid_map_start, config->gid_map_length);
    
    fd = open(map_path, O_WRONLY);
    if (fd == -1) {
        perror("open gid_map");
        return -1;
    }
    
    if (write(fd, map_content, strlen(map_content)) == -1) {
        perror("write gid_map");
        close(fd);
        return -1;
    }
    close(fd);
    
    return 0;
}

// Advanced cgroup v2 integration
int setup_enterprise_cgroups_v2(pid_t pid, struct resource_limits *limits) {
    char cgroup_path[512];
    char value_buf[64];
    int fd;
    
    // Create dedicated cgroup for the namespace
    snprintf(cgroup_path, sizeof(cgroup_path), 
             "/sys/fs/cgroup/enterprise-namespace-%d", pid);
    
    if (mkdir(cgroup_path, 0755) == -1 && errno != EEXIST) {
        perror("mkdir cgroup");
        return -1;
    }
    
    // Configure memory limits
    snprintf(cgroup_path, sizeof(cgroup_path),
             "/sys/fs/cgroup/enterprise-namespace-%d/memory.max", pid);
    fd = open(cgroup_path, O_WRONLY);
    if (fd != -1) {
        snprintf(value_buf, sizeof(value_buf), "%ld", limits->memory_max);
        write(fd, value_buf, strlen(value_buf));
        close(fd);
    }
    
    // Configure CPU limits
    snprintf(cgroup_path, sizeof(cgroup_path),
             "/sys/fs/cgroup/enterprise-namespace-%d/cpu.max", pid);
    fd = open(cgroup_path, O_WRONLY);
    if (fd != -1) {
        snprintf(value_buf, sizeof(value_buf), "%ld %ld", 
                 limits->cpu_quota, limits->cpu_period);
        write(fd, value_buf, strlen(value_buf));
        close(fd);
    }
    
    // Add process to cgroup
    snprintf(cgroup_path, sizeof(cgroup_path),
             "/sys/fs/cgroup/enterprise-namespace-%d/cgroup.procs", pid);
    fd = open(cgroup_path, O_WRONLY);
    if (fd != -1) {
        snprintf(value_buf, sizeof(value_buf), "%d", pid);
        write(fd, value_buf, strlen(value_buf));
        close(fd);
    }
    
    return 0;
}

Golang Namespace Management

// Enterprise namespace management in Go
package namespace

import (
    "fmt"
    "os"
    "os/exec"
    "path/filepath"
    "strconv"
    "syscall"
    "time"
    
    "github.com/containerd/cgroups/v3"
    "github.com/opencontainers/runc/libcontainer"
    "github.com/opencontainers/runc/libcontainer/configs"
)

type EnterpriseNamespaceManager struct {
    auditLogger     *AuditLogger
    resourceMonitor *ResourceMonitor
    securityPolicy  *SecurityPolicy
    cgroupManager   cgroups.Manager
}

type NamespaceConfig struct {
    ID              string
    Hostname        string
    DomainName      string
    RootFS          string
    UserMapping     []configs.IDMap
    GroupMapping    []configs.IDMap
    BindMounts      []BindMount
    NetworkConfig   *NetworkConfig
    ResourceLimits  *ResourceLimits
    SecurityOptions *SecurityOptions
}

type SecurityOptions struct {
    NoNewPrivileges bool
    Seccomp         *configs.Seccomp
    AppArmor        string
    SELinux         string
    Capabilities    []string
    ReadonlyPaths   []string
    MaskedPaths     []string
}

// Create enterprise namespace with comprehensive security
func (enm *EnterpriseNamespaceManager) CreateEnterpriseNamespace(
    config *NamespaceConfig) (*EnterpriseNamespace, error) {
    
    // Validate configuration
    if err := enm.validateConfig(config); err != nil {
        return nil, fmt.Errorf("invalid namespace config: %w", err)
    }
    
    // Create libcontainer configuration
    containerConfig := &configs.Config{
        Rootfs:      config.RootFS,
        Hostname:    config.Hostname,
        Domainname:  config.DomainName,
        UidMappings: config.UserMapping,
        GidMappings: config.GroupMapping,
        Namespaces: []configs.Namespace{
            {Type: configs.NEWNS},
            {Type: configs.NEWUTS},
            {Type: configs.NEWIPC},
            {Type: configs.NEWPID},
            {Type: configs.NEWNET},
            {Type: configs.NEWUSER},
            {Type: configs.NEWCGROUP},
            {Type: configs.NEWTIME}, // For time namespace isolation
        },
        Capabilities: enm.buildCapabilities(config.SecurityOptions),
        Networks:     enm.buildNetworkConfigs(config.NetworkConfig),
        Routes:       enm.buildRoutes(config.NetworkConfig),
    }
    
    // Configure cgroups v2
    if err := enm.configureCgroupsV2(containerConfig, config.ResourceLimits); err != nil {
        return nil, fmt.Errorf("failed to configure cgroups: %w", err)
    }
    
    // Configure security options
    enm.applySecurityOptions(containerConfig, config.SecurityOptions)
    
    // Configure bind mounts with security validation
    for _, mount := range config.BindMounts {
        if err := enm.validateBindMount(mount); err != nil {
            return nil, fmt.Errorf("invalid bind mount %s: %w", mount.Source, err)
        }
        
        containerConfig.Mounts = append(containerConfig.Mounts, &configs.Mount{
            Source:      mount.Source,
            Destination: mount.Destination,
            Device:      "bind",
            Flags:       mount.Flags | syscall.MS_BIND,
            Data:        mount.Data,
        })
    }
    
    // Create container factory
    factory, err := libcontainer.New("/var/lib/enterprise-containers",
        libcontainer.Cgroupfs,
        libcontainer.InitArgs(os.Args[0], "enterprise-init"))
    if err != nil {
        return nil, fmt.Errorf("failed to create container factory: %w", err)
    }
    
    // Create container
    container, err := factory.Create(config.ID, containerConfig)
    if err != nil {
        return nil, fmt.Errorf("failed to create container: %w", err)
    }
    
    // Create enterprise namespace wrapper
    namespace := &EnterpriseNamespace{
        ID:          config.ID,
        Container:   container,
        Config:      config,
        CreatedAt:   time.Now(),
        auditLogger: enm.auditLogger,
        monitor:     enm.resourceMonitor,
    }
    
    // Start monitoring
    go namespace.startResourceMonitoring()
    
    enm.auditLogger.LogNamespaceCreated(config.ID, config)
    return namespace, nil
}

// Configure cgroups v2 with enterprise resource management
func (enm *EnterpriseNamespaceManager) configureCgroupsV2(
    config *configs.Config, 
    limits *ResourceLimits) error {
    
    if limits == nil {
        return nil
    }
    
    // Memory configuration
    config.Cgroups.Resources.Memory = limits.Memory
    config.Cgroups.Resources.MemorySwap = limits.MemorySwap
    config.Cgroups.Resources.KernelMemory = limits.KernelMemory
    
    // CPU configuration
    config.Cgroups.Resources.CpuQuota = limits.CPUQuota
    config.Cgroups.Resources.CpuPeriod = limits.CPUPeriod
    config.Cgroups.Resources.CpuShares = limits.CPUShares
    config.Cgroups.Resources.CpusetCpus = limits.CPUSet
    
    // I/O configuration
    config.Cgroups.Resources.BlkioWeight = limits.BlkIOWeight
    config.Cgroups.Resources.BlkioDeviceReadBps = limits.BlkIOReadBps
    config.Cgroups.Resources.BlkioDeviceWriteBps = limits.BlkIOWriteBps
    
    // Network configuration
    config.Cgroups.Resources.NetClsClassid = limits.NetClsClassID
    config.Cgroups.Resources.NetPrioIfpriomap = limits.NetPrioMap
    
    // PID limits
    config.Cgroups.Resources.PidsLimit = limits.PidsLimit
    
    return nil
}

// Enterprise namespace with monitoring and lifecycle management
type EnterpriseNamespace struct {
    ID          string
    Container   libcontainer.Container
    Config      *NamespaceConfig
    CreatedAt   time.Time
    Status      NamespaceStatus
    auditLogger *AuditLogger
    monitor     *ResourceMonitor
    
    // Runtime metrics
    metrics struct {
        CPUUsage    float64
        MemoryUsage int64
        NetworkIO   NetworkStats
        DiskIO      DiskStats
    }
}

// Start the namespace with enterprise monitoring
func (en *EnterpriseNamespace) Start(process *configs.Process) error {
    // Validate process configuration
    if err := en.validateProcess(process); err != nil {
        return fmt.Errorf("invalid process config: %w", err)
    }
    
    // Apply additional security restrictions
    en.applyRuntimeSecurity(process)
    
    // Start the container process
    if err := en.Container.Start(process); err != nil {
        en.auditLogger.LogNamespaceStartFailed(en.ID, err)
        return fmt.Errorf("failed to start namespace process: %w", err)
    }
    
    en.Status = StatusRunning
    en.auditLogger.LogNamespaceStarted(en.ID)
    
    // Start health monitoring
    go en.startHealthMonitoring()
    
    return nil
}

// Resource monitoring for enterprise namespaces
func (en *EnterpriseNamespace) startResourceMonitoring() {
    ticker := time.NewTicker(10 * time.Second)
    defer ticker.Stop()
    
    for {
        select {
        case <-ticker.C:
            stats, err := en.Container.Stats()
            if err != nil {
                en.auditLogger.LogMonitoringError(en.ID, err)
                continue
            }
            
            // Update metrics
            en.metrics.CPUUsage = float64(stats.CgroupStats.CpuStats.CpuUsage.TotalUsage)
            en.metrics.MemoryUsage = int64(stats.CgroupStats.MemoryStats.Usage.Usage)
            
            // Check for resource violations
            if en.checkResourceViolations(stats) {
                en.handleResourceViolation(stats)
            }
            
            // Send metrics to monitoring system
            en.monitor.RecordNamespaceMetrics(en.ID, stats)
            
        case <-en.Container.Done():
            return
        }
    }
}

// Health monitoring with automatic recovery
func (en *EnterpriseNamespace) startHealthMonitoring() {
    ticker := time.NewTicker(30 * time.Second)
    defer ticker.Stop()
    
    for {
        select {
        case <-ticker.C:
            if err := en.performHealthCheck(); err != nil {
                en.auditLogger.LogHealthCheckFailed(en.ID, err)
                
                // Attempt automatic recovery
                if en.Config.AutoRecover {
                    if err := en.attemptRecovery(); err != nil {
                        en.auditLogger.LogRecoveryFailed(en.ID, err)
                    }
                }
            }
            
        case <-en.Container.Done():
            return
        }
    }
}

type NamespaceStatus int

const (
    StatusCreated NamespaceStatus = iota
    StatusRunning
    StatusStopped
    StatusFailed
    StatusRecovering
)

type ResourceLimits struct {
    Memory          int64
    MemorySwap      int64
    KernelMemory    int64
    CPUQuota        int64
    CPUPeriod       uint64
    CPUShares       uint64
    CPUSet          string
    BlkIOWeight     uint16
    BlkIOReadBps    []*configs.ThrottleDevice
    BlkIOWriteBps   []*configs.ThrottleDevice
    NetClsClassID   uint32
    NetPrioMap      []*configs.IfPrioMap
    PidsLimit       int64
}

Chapter 3: Enterprise Security Management

Advanced SELinux Administration

#!/bin/bash
# Enterprise SELinux policy management

# Custom SELinux policy for enterprise applications
create_enterprise_selinux_policy() {
    local app_name="$1"
    local policy_dir="/etc/selinux/local/enterprise"
    
    mkdir -p "$policy_dir"
    
    # Create type enforcement file
    cat > "$policy_dir/${app_name}.te" << EOF
module ${app_name} 1.0;

require {
    type unconfined_t;
    type user_home_t;
    type httpd_t;
    type httpd_exec_t;
    type var_log_t;
    type var_lib_t;
    class file { read write create unlink };
    class dir { search add_name remove_name };
    class process { transition };
}

# Define custom types for the application
type ${app_name}_t;
type ${app_name}_exec_t;
type ${app_name}_log_t;
type ${app_name}_var_lib_t;

# File contexts
files_type(${app_name}_exec_t)
logging_log_file(${app_name}_log_t)
files_type(${app_name}_var_lib_t)

# Domain transition
domain_type(${app_name}_t)
domain_entry_file(${app_name}_t, ${app_name}_exec_t)

# Allow domain transition from unconfined_t
allow unconfined_t ${app_name}_exec_t:file { read execute };
allow unconfined_t ${app_name}_t:process transition;

# Application permissions
allow ${app_name}_t ${app_name}_exec_t:file execute;
allow ${app_name}_t ${app_name}_log_t:file { create write append };
allow ${app_name}_t ${app_name}_var_lib_t:file { read write create unlink };
allow ${app_name}_t ${app_name}_var_lib_t:dir { search add_name remove_name };

# Network permissions for enterprise apps
allow ${app_name}_t self:tcp_socket { create connect listen accept };
allow ${app_name}_t self:udp_socket { create connect };

# Enterprise security constraints
constrain file { read write } (
    u1 == u2 or 
    (r1 == enterprise_admin_r and r2 == enterprise_user_r)
);
EOF

    # Create file contexts
    cat > "$policy_dir/${app_name}.fc" << EOF
/opt/${app_name}/bin/${app_name}    --  gen_context(system_u:object_r:${app_name}_exec_t,s0)
/opt/${app_name}/lib(/.*)?              gen_context(system_u:object_r:${app_name}_var_lib_t,s0)
/var/log/${app_name}(/.*)?              gen_context(system_u:object_r:${app_name}_log_t,s0)
EOF

    # Compile and install policy
    cd "$policy_dir"
    make -f /usr/share/selinux/devel/Makefile ${app_name}.pp
    semodule -i ${app_name}.pp
    
    # Apply file contexts
    restorecon -R /opt/${app_name}
    restorecon -R /var/log/${app_name}
    
    log_info "SELinux policy created and installed for $app_name"
}

# Enterprise audit log analysis
analyze_selinux_denials() {
    local log_file="/var/log/audit/audit.log"
    local output_dir="/var/log/selinux-analysis"
    local date_filter="$(date '+%Y-%m-%d')"
    
    mkdir -p "$output_dir"
    
    # Extract today's denials
    ausearch -m avc -ts today | audit2allow -a > "$output_dir/denials-${date_filter}.log"
    
    # Generate policy suggestions
    ausearch -m avc -ts today | audit2allow -M enterprise-policy-$(date +%s)
    
    # Analyze denial patterns
    python3 << 'EOF'
import re
import json
from collections import defaultdict
import sys

denial_patterns = defaultdict(int)
source_contexts = defaultdict(int)
target_contexts = defaultdict(int)

with open('/var/log/audit/audit.log', 'r') as f:
    for line in f:
        if 'avc:  denied' in line and '$(date '+%Y-%m-%d')' in line:
            # Extract source and target contexts
            scontext_match = re.search(r'scontext=([^\\s]+)', line)
            tcontext_match = re.search(r'tcontext=([^\\s]+)', line)
            
            if scontext_match and tcontext_match:
                scontext = scontext_match.group(1)
                tcontext = tcontext_match.group(1)
                
                source_contexts[scontext] += 1
                target_contexts[tcontext] += 1
                
                pattern = f"{scontext} -> {tcontext}"
                denial_patterns[pattern] += 1

# Generate report
report = {
    'date': '$(date '+%Y-%m-%d')',
    'total_denials': sum(denial_patterns.values()),
    'top_denial_patterns': dict(sorted(denial_patterns.items(), 
                                     key=lambda x: x[1], reverse=True)[:10]),
    'top_source_contexts': dict(sorted(source_contexts.items(), 
                                     key=lambda x: x[1], reverse=True)[:10]),
    'top_target_contexts': dict(sorted(target_contexts.items(), 
                                     key=lambda x: x[1], reverse=True)[:10])
}

with open('/var/log/selinux-analysis/analysis-$(date +%Y-%m-%d).json', 'w') as f:
    json.dump(report, f, indent=2)

print(f"Analysis complete. Found {report['total_denials']} denials.")
EOF
}

# Advanced AppArmor profile generation
generate_enterprise_apparmor_profile() {
    local app_name="$1"
    local app_path="$2"
    local profile_path="/etc/apparmor.d/${app_name}"
    
    cat > "$profile_path" << EOF
#include <tunables/global>

${app_path} {
  #include <abstractions/base>
  #include <abstractions/nameservice>
  #include <abstractions/openssl>
  #include <abstractions/ssl_certs>
  
  # Enterprise capabilities
  capability net_bind_service,
  capability setuid,
  capability setgid,
  capability dac_override,
  capability sys_resource,
  
  # Network access
  network tcp,
  network udp,
  
  # File system access
  /opt/${app_name}/** r,
  /opt/${app_name}/bin/${app_name} ix,
  /opt/${app_name}/lib/** mr,
  /opt/${app_name}/etc/** r,
  
  # Logs and data
  /var/log/${app_name}/** rw,
  /var/lib/${app_name}/** rw,
  /tmp/${app_name}_* rw,
  
  # System libraries
  /lib/x86_64-linux-gnu/** mr,
  /usr/lib/x86_64-linux-gnu/** mr,
  
  # Proc and sys access (limited)
  @{PROC}/sys/kernel/random/uuid r,
  @{PROC}/meminfo r,
  @{PROC}/cpuinfo r,
  
  # Deny dangerous operations
  deny /etc/shadow r,
  deny /etc/passwd w,
  deny /etc/group w,
  deny /proc/*/mem rw,
  deny /sys/kernel/security/** rw,
  
  # Enterprise-specific rules
  owner /home/*/.${app_name}/** rw,
  /etc/${app_name}/** r,
  
  # Signal handling
  signal receive set=(term, kill, usr1, usr2),
  signal send set=(term, kill) peer=${app_name},
}
EOF

    # Load the profile
    apparmor_parser -r "$profile_path"
    
    # Enable enforcement
    aa-enforce "$profile_path"
    
    log_info "AppArmor profile created and enforced for $app_name"
}

Kubernetes RBAC Enterprise Integration

# Advanced RBAC for enterprise Kubernetes
apiVersion: v1
kind: Namespace
metadata:
  name: enterprise-production
  labels:
    security-tier: "high"
    compliance: "sox-pci"
    environment: "production"
  annotations:
    security.enterprise.com/network-policy: "strict"
    security.enterprise.com/pod-security: "restricted"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: enterprise-admin
  labels:
    rbac.enterprise.com/role-type: "administrative"
rules:
# Full access to enterprise namespaces
- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]
  resourceNames: []
# Restricted cluster-level operations
- apiGroups: ["rbac.authorization.k8s.io"]
  resources: ["clusterroles", "clusterrolebindings"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
  resources: ["storageclasses"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]
# Custom resource management
- apiGroups: ["enterprise.com"]
  resources: ["enterpriseconfigs", "securitypolicies"]
  verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: enterprise-developer
  labels:
    rbac.enterprise.com/role-type: "development"
rules:
# Application management in designated namespaces
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "daemonsets"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["pods", "pods/log", "pods/exec"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
  resources: ["configmaps", "secrets"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  resourceNames: ["app-*", "config-*"]  # Only app-specific resources
# Service mesh integration
- apiGroups: ["networking.istio.io"]
  resources: ["virtualservices", "destinationrules"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
# Monitoring access
- apiGroups: ["monitoring.coreos.com"]
  resources: ["servicemonitors", "prometheusrules"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: enterprise-auditor
  labels:
    rbac.enterprise.com/role-type: "compliance"
rules:
# Read-only access for compliance auditing
- apiGroups: [""]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions", "networking.k8s.io"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["rbac.authorization.k8s.io"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
# Audit log access
- apiGroups: ["audit.k8s.io"]
  resources: ["events"]
  verbs: ["get", "list", "watch"]
# Security policy review
- apiGroups: ["security.openshift.io", "policy"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
---
# Advanced RBAC with attribute-based access control
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: enterprise-service-account-manager
rules:
- apiGroups: [""]
  resources: ["serviceaccounts"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  resourceNames: ["enterprise-*"]  # Only enterprise service accounts
- apiGroups: [""]
  resources: ["serviceaccounts/token"]
  verbs: ["create"]
  resourceNames: ["enterprise-*"]
# OIDC token management
- apiGroups: ["authentication.k8s.io"]
  resources: ["tokenreviews"]
  verbs: ["create"]
- apiGroups: ["authorization.k8s.io"]  
  resources: ["subjectaccessreviews"]
  verbs: ["create"]
---
# Enterprise namespace role binding with conditions
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  namespace: enterprise-production
  name: enterprise-admin-binding
  labels:
    environment: "production"
    security-tier: "high"
subjects:
- kind: User
  name: admin@enterprise.com
  apiGroup: rbac.authorization.k8s.io
- kind: Group
  name: enterprise-admin-group
  apiGroup: rbac.authorization.k8s.io
- kind: ServiceAccount
  name: enterprise-admin-sa
  namespace: enterprise-production
roleRef:
  kind: ClusterRole
  name: enterprise-admin
  apiGroup: rbac.authorization.k8s.io
---
# Pod Security Policy for enterprise workloads
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: enterprise-restricted
  labels:
    security.enterprise.com/policy-level: "restricted"
spec:
  privileged: false
  allowPrivilegeEscalation: false
  requiredDropCapabilities:
    - ALL
  allowedCapabilities:
    - NET_BIND_SERVICE
  volumes:
    - 'configMap'
    - 'emptyDir'
    - 'projected'
    - 'secret'
    - 'downwardAPI'
    - 'persistentVolumeClaim'
  hostNetwork: false
  hostIPC: false
  hostPID: false
  runAsUser:
    rule: 'MustRunAsNonRoot'
  seLinux:
    rule: 'RunAsAny'
  seccompProfile:
    type: 'RuntimeDefault'
  fsGroup:
    rule: 'RunAsAny'
  readOnlyRootFilesystem: true
  # Resource limits enforcement
  defaultAllowPrivilegeEscalation: false
  forbiddenSysctls:
    - '*'
  allowedUnsafeSysctls: []
  # Network restrictions
  defaultAddCapabilities: []
  requiredDropCapabilities:
    - ALL
  allowedCapabilities:
    - NET_BIND_SERVICE
    - SYS_TIME  # For NTP synchronization
---
# Custom admission controller webhook
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionWebhook
metadata:
  name: enterprise-policy-validator
webhooks:
- name: security-policy.enterprise.com
  clientConfig:
    service:
      name: enterprise-admission-webhook
      namespace: enterprise-system
      path: "/validate-security"
  rules:
  - operations: ["CREATE", "UPDATE"]
    apiGroups: [""]
    apiVersions: ["v1"]
    resources: ["pods"]
  - operations: ["CREATE", "UPDATE"]
    apiGroups: ["apps"]
    apiVersions: ["v1"]
    resources: ["deployments", "daemonsets", "statefulsets"]
  admissionReviewVersions: ["v1", "v1beta1"]
  sideEffects: None
  failurePolicy: Fail
  # Enterprise-specific admission rules
  namespaceSelector:
    matchLabels:
      security-tier: "high"

This comprehensive guide continues with advanced topics in enterprise Linux systems administration. Would you like me to continue with the remaining sections covering automation, monitoring, disaster recovery, and career development frameworks?