param( [string]$EvidenceDate = (Get-Date -Format 'yyyy-MM-dd'), [string]$EnvFilePath = '', [int]$TimeoutSeconds = 20, [switch]$DisableSsl ) $ErrorActionPreference = 'Stop' $projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path $evidenceRoot = Join-Path $projectRoot "docs\evidence\ops\$EvidenceDate\alerting" $timestamp = Get-Date -Format 'yyyyMMdd-HHmmss' $drillRoot = Join-Path $evidenceRoot $timestamp $sanitizedConfigPath = Join-Path $drillRoot 'alertmanager.live.redacted.yaml' $reportPath = Join-Path $drillRoot 'ALERTMANAGER_LIVE_DELIVERY_DRILL.md' $tempRenderedPath = Join-Path ([System.IO.Path]::GetTempPath()) ("alertmanager-live-" + [System.Guid]::NewGuid().ToString('N') + '.yaml') $requiredVariables = @( 'ALERTMANAGER_DEFAULT_TO', 'ALERTMANAGER_CRITICAL_TO', 'ALERTMANAGER_WARNING_TO', 'ALERTMANAGER_FROM', 'ALERTMANAGER_SMARTHOST', 'ALERTMANAGER_AUTH_USERNAME', 'ALERTMANAGER_AUTH_PASSWORD' ) New-Item -ItemType Directory -Force $evidenceRoot, $drillRoot | Out-Null function Import-EnvFileToProcess { param( [Parameter(Mandatory = $true)][string]$Path ) $saved = @() foreach ($rawLine in Get-Content $Path -Encoding UTF8) { $line = $rawLine.Trim() if ($line -eq '' -or $line.StartsWith('#')) { continue } $parts = $line -split '=', 2 if ($parts.Count -ne 2) { throw "invalid env line: $line" } $name = $parts[0].Trim() $value = $parts[1].Trim() $existing = [Environment]::GetEnvironmentVariable($name, 'Process') $saved += [pscustomobject]@{ Name = $name HadValue = -not [string]::IsNullOrEmpty($existing) Value = $existing } [Environment]::SetEnvironmentVariable($name, $value, 'Process') } return $saved } function Restore-ProcessEnv { param( [Parameter(Mandatory = $true)][object[]]$SavedState ) foreach ($entry in $SavedState) { if ($entry.HadValue) { [Environment]::SetEnvironmentVariable($entry.Name, $entry.Value, 'Process') continue } Remove-Item ("Env:" + $entry.Name) -ErrorAction SilentlyContinue } } function Get-ConfiguredValues { param( [Parameter(Mandatory = $true)][string[]]$Names ) $values = @{} foreach ($name in $Names) { $values[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') } return $values } function Get-PlaceholderFindings { param( [Parameter(Mandatory = $true)][hashtable]$Values ) $findings = @() foreach ($entry in $Values.GetEnumerator()) { $name = $entry.Key $value = [string]$entry.Value if ([string]::IsNullOrWhiteSpace($value)) { continue } if ($value -match '\$\{[A-Z0-9_]+\}') { $findings += "$name contains unresolved placeholder syntax" } if ($value -match '(?i)\bexample\.(com|org)\b') { $findings += "$name still uses example domain" } if ($name -like '*PASSWORD' -and $value -match '(?i)^(replace-with-secret|synthetic-secret-for-render-drill|password)$') { $findings += "$name still uses placeholder secret" } } return $findings } function Parse-Smarthost { param( [Parameter(Mandatory = $true)][string]$Value ) $match = [regex]::Match($Value, '^(?\[[^\]]+\]|[^:]+)(:(?\d+))?$') if (-not $match.Success) { throw "invalid ALERTMANAGER_SMARTHOST value: $Value" } $host = $match.Groups['host'].Value.Trim('[', ']') $port = if ($match.Groups['port'].Success) { [int]$match.Groups['port'].Value } else { 25 } return [pscustomobject]@{ Host = $host Port = $port Raw = $Value } } function Split-Recipients { param( [Parameter(Mandatory = $true)][string]$Value ) return @( $Value -split '[,;]' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne '' } ) } function Mask-EmailList { param( [Parameter(Mandatory = $true)][string]$Value ) $masked = @() foreach ($recipient in Split-Recipients -Value $Value) { if ($recipient -notmatch '^(?[^@]+)@(?.+)$') { $masked += '***REDACTED***' continue } $local = $Matches['local'] $domain = $Matches['domain'] $prefix = if ($local.Length -gt 0) { $local.Substring(0, 1) } else { '*' } $masked += ($prefix + '***@' + $domain) } return $masked -join ', ' } function Mask-Host { param( [Parameter(Mandatory = $true)][string]$Value ) if ([string]::IsNullOrWhiteSpace($Value)) { return '***REDACTED_HOST***' } if ($Value.Length -le 3) { return ($Value.Substring(0, 1) + '**') } return ($Value.Substring(0, 1) + '***' + $Value.Substring($Value.Length - 2)) } function Test-TcpConnectivity { param( [Parameter(Mandatory = $true)][string]$Host, [Parameter(Mandatory = $true)][int]$Port, [Parameter(Mandatory = $true)][int]$TimeoutSeconds ) $client = New-Object System.Net.Sockets.TcpClient try { $asyncResult = $client.BeginConnect($Host, $Port, $null, $null) if (-not $asyncResult.AsyncWaitHandle.WaitOne($TimeoutSeconds * 1000, $false)) { throw "tcp connect timeout after ${TimeoutSeconds}s" } $client.EndConnect($asyncResult) return [pscustomobject]@{ Succeeded = $true Error = '' } } catch { return [pscustomobject]@{ Succeeded = $false Error = $_.Exception.Message } } finally { $client.Dispose() } } function Send-SmtpMessage { param( [Parameter(Mandatory = $true)][pscustomobject]$Smarthost, [Parameter(Mandatory = $true)][string]$From, [Parameter(Mandatory = $true)][string]$To, [Parameter(Mandatory = $true)][string]$Username, [Parameter(Mandatory = $true)][string]$Password, [Parameter(Mandatory = $true)][string]$RouteName, [Parameter(Mandatory = $true)][int]$TimeoutSeconds, [Parameter(Mandatory = $true)][bool]$EnableSsl ) $message = [System.Net.Mail.MailMessage]::new() $smtp = [System.Net.Mail.SmtpClient]::new($Smarthost.Host, $Smarthost.Port) try { $message.From = [System.Net.Mail.MailAddress]::new($From) foreach ($recipient in Split-Recipients -Value $To) { $message.To.Add($recipient) } $message.Subject = "[ALERTING-LIVE-DRILL][$RouteName] $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')" $message.Body = @" This is a live alert delivery drill. Route: $RouteName Project: $projectRoot GeneratedAt: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz') "@ $smtp.EnableSsl = $EnableSsl $smtp.Timeout = $TimeoutSeconds * 1000 $smtp.DeliveryMethod = [System.Net.Mail.SmtpDeliveryMethod]::Network $smtp.UseDefaultCredentials = $false $smtp.Credentials = [System.Net.NetworkCredential]::new($Username, $Password) $smtp.Send($message) return [pscustomobject]@{ Route = $RouteName RecipientMask = Mask-EmailList -Value $To Accepted = $true Error = '' } } catch { return [pscustomobject]@{ Route = $RouteName RecipientMask = Mask-EmailList -Value $To Accepted = $false Error = $_.Exception.Message } } finally { $message.Dispose() $smtp.Dispose() } } function Get-RedactedRenderedConfig { param( [Parameter(Mandatory = $true)][string]$RenderedContent, [Parameter(Mandatory = $true)][hashtable]$Values ) $redacted = $RenderedContent $replacementMap = @{ 'ALERTMANAGER_DEFAULT_TO' = '***REDACTED_DEFAULT_TO***' 'ALERTMANAGER_CRITICAL_TO' = '***REDACTED_CRITICAL_TO***' 'ALERTMANAGER_WARNING_TO' = '***REDACTED_WARNING_TO***' 'ALERTMANAGER_FROM' = '***REDACTED_FROM***' 'ALERTMANAGER_SMARTHOST' = '***REDACTED_SMARTHOST***' 'ALERTMANAGER_AUTH_USERNAME' = '***REDACTED_AUTH_USERNAME***' 'ALERTMANAGER_AUTH_PASSWORD' = '***REDACTED_AUTH_PASSWORD***' } foreach ($entry in $replacementMap.GetEnumerator()) { $value = [string]$Values[$entry.Key] if ([string]::IsNullOrWhiteSpace($value)) { continue } $redacted = [regex]::Replace($redacted, [regex]::Escape($value), [System.Text.RegularExpressions.MatchEvaluator]{ param($m) $entry.Value }) } return $redacted } $savedEnvState = @() $values = @{} $missingVariables = @() $placeholderFindings = @() $renderSucceeded = $false $tcpResult = [pscustomobject]@{ Succeeded = $false; Error = 'not-run' } $sendResults = @() $success = $false $failureReason = '' $smarthost = $null $envSource = if ([string]::IsNullOrWhiteSpace($EnvFilePath)) { 'process environment' } else { $EnvFilePath } try { if (-not [string]::IsNullOrWhiteSpace($EnvFilePath)) { if (-not (Test-Path $EnvFilePath)) { throw "env file not found: $EnvFilePath" } $savedEnvState = Import-EnvFileToProcess -Path $EnvFilePath } $values = Get-ConfiguredValues -Names $requiredVariables $missingVariables = @( $requiredVariables | Where-Object { [string]::IsNullOrWhiteSpace([string]$values[$_]) } ) $placeholderFindings = Get-PlaceholderFindings -Values $values if ($missingVariables.Count -gt 0) { throw "missing required alertmanager variables: $($missingVariables -join ', ')" } if ($placeholderFindings.Count -gt 0) { throw "placeholder or example values detected" } $smarthost = Parse-Smarthost -Value ([string]$values['ALERTMANAGER_SMARTHOST']) & (Join-Path $PSScriptRoot 'render-alertmanager-config.ps1') ` -TemplatePath (Join-Path $projectRoot 'deployment\alertmanager\alertmanager.yml') ` -OutputPath $tempRenderedPath ` -EnvFilePath $EnvFilePath | Out-Null $renderedContent = Get-Content $tempRenderedPath -Raw -Encoding UTF8 $redactedContent = Get-RedactedRenderedConfig -RenderedContent $renderedContent -Values $values Set-Content -Path $sanitizedConfigPath -Value $redactedContent -Encoding UTF8 $renderSucceeded = $true $tcpResult = Test-TcpConnectivity -Host $smarthost.Host -Port $smarthost.Port -TimeoutSeconds $TimeoutSeconds if (-not $tcpResult.Succeeded) { throw "smtp tcp connectivity failed: $($tcpResult.Error)" } $routes = @( [pscustomobject]@{ Name = 'default'; To = [string]$values['ALERTMANAGER_DEFAULT_TO'] } [pscustomobject]@{ Name = 'critical-alerts'; To = [string]$values['ALERTMANAGER_CRITICAL_TO'] } [pscustomobject]@{ Name = 'warning-alerts'; To = [string]$values['ALERTMANAGER_WARNING_TO'] } ) foreach ($route in $routes) { $sendResults += Send-SmtpMessage ` -Smarthost $smarthost ` -From ([string]$values['ALERTMANAGER_FROM']) ` -To $route.To ` -Username ([string]$values['ALERTMANAGER_AUTH_USERNAME']) ` -Password ([string]$values['ALERTMANAGER_AUTH_PASSWORD']) ` -RouteName $route.Name ` -TimeoutSeconds $TimeoutSeconds ` -EnableSsl (-not $DisableSsl.IsPresent) } $failedRoutes = @($sendResults | Where-Object { -not $_.Accepted }) if ($failedRoutes.Count -gt 0) { throw "smtp send failed for route(s): $($failedRoutes.Route -join ', ')" } $success = $true } catch { $failureReason = $_.Exception.Message } finally { if (Test-Path $tempRenderedPath) { Remove-Item $tempRenderedPath -Force -ErrorAction SilentlyContinue } if ($savedEnvState.Count -gt 0) { Restore-ProcessEnv -SavedState $savedEnvState } $reportLines = @( '# Alertmanager Live Delivery Drill', '', "- Generated at: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')", "- Template file: $(Join-Path $projectRoot 'deployment\alertmanager\alertmanager.yml')", "- Env source: $envSource", "- Redacted rendered config: $(if (Test-Path $sanitizedConfigPath) { $sanitizedConfigPath } else { 'not-generated' })", '', '## Strict Preconditions', '', "- Required variables present: $($missingVariables.Count -eq 0)", "- Placeholder/example-value findings: $(if ($placeholderFindings.Count -gt 0) { $placeholderFindings -join '; ' } else { 'none' })", "- Render path succeeded: $renderSucceeded", '', '## Delivery Attempt', '', "- SMTP host: $(if ($smarthost) { (Mask-Host -Value $smarthost.Host) } else { 'unparsed' })", "- SMTP port: $(if ($smarthost) { $smarthost.Port } else { 'unparsed' })", "- TLS enabled: $(-not $DisableSsl.IsPresent)", "- TCP connectivity succeeded: $($tcpResult.Succeeded)", "- TCP connectivity error: $(if ($tcpResult.Error) { $tcpResult.Error } else { 'none' })", '' ) if ($sendResults.Count -gt 0) { $reportLines += '## Route Results' $reportLines += '' foreach ($result in $sendResults) { $reportLines += "- Route $($result.Route): accepted=$($result.Accepted), recipients=$($result.RecipientMask), error=$(if ([string]::IsNullOrWhiteSpace($result.Error)) { 'none' } else { $result.Error })" } $reportLines += '' } $reportLines += '## Conclusion' $reportLines += '' $reportLines += "- Live external delivery closed: $success" $reportLines += "- Failure reason: $(if ([string]::IsNullOrWhiteSpace($failureReason)) { 'none' } else { $failureReason })" $reportLines += '- This drill fails closed on unresolved placeholders, example domains, and placeholder secrets.' $reportLines += '- The evidence intentionally stores only redacted config output and masked recipient information.' $reportLines += '- A successful run proves real secret injection plus SMTP server acceptance for the configured on-call routes; it does not by itself prove downstream human acknowledgment.' $reportLines += '' Set-Content -Path $reportPath -Value ($reportLines -join [Environment]::NewLine) -Encoding UTF8 Get-Content $reportPath } if (-not $success) { throw "alertmanager live delivery drill failed: $failureReason" }