437 lines
13 KiB
PowerShell
437 lines
13 KiB
PowerShell
param(
|
|
[string]$EvidenceDate = (Get-Date -Format 'yyyy-MM-dd'),
|
|
[string]$EnvFilePath = '',
|
|
[int]$TimeoutSeconds = 20,
|
|
[switch]$DisableSsl
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
|
$evidenceRoot = Join-Path $projectRoot "docs\evidence\ops\$EvidenceDate\alerting"
|
|
$timestamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
|
$drillRoot = Join-Path $evidenceRoot $timestamp
|
|
$sanitizedConfigPath = Join-Path $drillRoot 'alertmanager.live.redacted.yaml'
|
|
$reportPath = Join-Path $drillRoot 'ALERTMANAGER_LIVE_DELIVERY_DRILL.md'
|
|
$tempRenderedPath = Join-Path ([System.IO.Path]::GetTempPath()) ("alertmanager-live-" + [System.Guid]::NewGuid().ToString('N') + '.yaml')
|
|
|
|
$requiredVariables = @(
|
|
'ALERTMANAGER_DEFAULT_TO',
|
|
'ALERTMANAGER_CRITICAL_TO',
|
|
'ALERTMANAGER_WARNING_TO',
|
|
'ALERTMANAGER_FROM',
|
|
'ALERTMANAGER_SMARTHOST',
|
|
'ALERTMANAGER_AUTH_USERNAME',
|
|
'ALERTMANAGER_AUTH_PASSWORD'
|
|
)
|
|
|
|
New-Item -ItemType Directory -Force $evidenceRoot, $drillRoot | Out-Null
|
|
|
|
function Import-EnvFileToProcess {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Path
|
|
)
|
|
|
|
$saved = @()
|
|
foreach ($rawLine in Get-Content $Path -Encoding UTF8) {
|
|
$line = $rawLine.Trim()
|
|
if ($line -eq '' -or $line.StartsWith('#')) {
|
|
continue
|
|
}
|
|
|
|
$parts = $line -split '=', 2
|
|
if ($parts.Count -ne 2) {
|
|
throw "invalid env line: $line"
|
|
}
|
|
|
|
$name = $parts[0].Trim()
|
|
$value = $parts[1].Trim()
|
|
$existing = [Environment]::GetEnvironmentVariable($name, 'Process')
|
|
$saved += [pscustomobject]@{
|
|
Name = $name
|
|
HadValue = -not [string]::IsNullOrEmpty($existing)
|
|
Value = $existing
|
|
}
|
|
[Environment]::SetEnvironmentVariable($name, $value, 'Process')
|
|
}
|
|
|
|
return $saved
|
|
}
|
|
|
|
function Restore-ProcessEnv {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object[]]$SavedState
|
|
)
|
|
|
|
foreach ($entry in $SavedState) {
|
|
if ($entry.HadValue) {
|
|
[Environment]::SetEnvironmentVariable($entry.Name, $entry.Value, 'Process')
|
|
continue
|
|
}
|
|
|
|
Remove-Item ("Env:" + $entry.Name) -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
function Get-ConfiguredValues {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string[]]$Names
|
|
)
|
|
|
|
$values = @{}
|
|
foreach ($name in $Names) {
|
|
$values[$name] = [Environment]::GetEnvironmentVariable($name, 'Process')
|
|
}
|
|
return $values
|
|
}
|
|
|
|
function Get-PlaceholderFindings {
|
|
param(
|
|
[Parameter(Mandatory = $true)][hashtable]$Values
|
|
)
|
|
|
|
$findings = @()
|
|
foreach ($entry in $Values.GetEnumerator()) {
|
|
$name = $entry.Key
|
|
$value = [string]$entry.Value
|
|
|
|
if ([string]::IsNullOrWhiteSpace($value)) {
|
|
continue
|
|
}
|
|
|
|
if ($value -match '\$\{[A-Z0-9_]+\}') {
|
|
$findings += "$name contains unresolved placeholder syntax"
|
|
}
|
|
|
|
if ($value -match '(?i)\bexample\.(com|org)\b') {
|
|
$findings += "$name still uses example domain"
|
|
}
|
|
|
|
if ($name -like '*PASSWORD' -and $value -match '(?i)^(replace-with-secret|synthetic-secret-for-render-drill|password)$') {
|
|
$findings += "$name still uses placeholder secret"
|
|
}
|
|
}
|
|
|
|
return $findings
|
|
}
|
|
|
|
function Parse-Smarthost {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Value
|
|
)
|
|
|
|
$match = [regex]::Match($Value, '^(?<host>\[[^\]]+\]|[^:]+)(:(?<port>\d+))?$')
|
|
if (-not $match.Success) {
|
|
throw "invalid ALERTMANAGER_SMARTHOST value: $Value"
|
|
}
|
|
|
|
$host = $match.Groups['host'].Value.Trim('[', ']')
|
|
$port = if ($match.Groups['port'].Success) { [int]$match.Groups['port'].Value } else { 25 }
|
|
|
|
return [pscustomobject]@{
|
|
Host = $host
|
|
Port = $port
|
|
Raw = $Value
|
|
}
|
|
}
|
|
|
|
function Split-Recipients {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Value
|
|
)
|
|
|
|
return @(
|
|
$Value -split '[,;]' |
|
|
ForEach-Object { $_.Trim() } |
|
|
Where-Object { $_ -ne '' }
|
|
)
|
|
}
|
|
|
|
function Mask-EmailList {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Value
|
|
)
|
|
|
|
$masked = @()
|
|
foreach ($recipient in Split-Recipients -Value $Value) {
|
|
if ($recipient -notmatch '^(?<local>[^@]+)@(?<domain>.+)$') {
|
|
$masked += '***REDACTED***'
|
|
continue
|
|
}
|
|
|
|
$local = $Matches['local']
|
|
$domain = $Matches['domain']
|
|
$prefix = if ($local.Length -gt 0) { $local.Substring(0, 1) } else { '*' }
|
|
$masked += ($prefix + '***@' + $domain)
|
|
}
|
|
|
|
return $masked -join ', '
|
|
}
|
|
|
|
function Mask-Host {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Value
|
|
)
|
|
|
|
if ([string]::IsNullOrWhiteSpace($Value)) {
|
|
return '***REDACTED_HOST***'
|
|
}
|
|
|
|
if ($Value.Length -le 3) {
|
|
return ($Value.Substring(0, 1) + '**')
|
|
}
|
|
|
|
return ($Value.Substring(0, 1) + '***' + $Value.Substring($Value.Length - 2))
|
|
}
|
|
|
|
function Test-TcpConnectivity {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Host,
|
|
[Parameter(Mandatory = $true)][int]$Port,
|
|
[Parameter(Mandatory = $true)][int]$TimeoutSeconds
|
|
)
|
|
|
|
$client = New-Object System.Net.Sockets.TcpClient
|
|
try {
|
|
$asyncResult = $client.BeginConnect($Host, $Port, $null, $null)
|
|
if (-not $asyncResult.AsyncWaitHandle.WaitOne($TimeoutSeconds * 1000, $false)) {
|
|
throw "tcp connect timeout after ${TimeoutSeconds}s"
|
|
}
|
|
|
|
$client.EndConnect($asyncResult)
|
|
return [pscustomobject]@{
|
|
Succeeded = $true
|
|
Error = ''
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
Succeeded = $false
|
|
Error = $_.Exception.Message
|
|
}
|
|
} finally {
|
|
$client.Dispose()
|
|
}
|
|
}
|
|
|
|
function Send-SmtpMessage {
|
|
param(
|
|
[Parameter(Mandatory = $true)][pscustomobject]$Smarthost,
|
|
[Parameter(Mandatory = $true)][string]$From,
|
|
[Parameter(Mandatory = $true)][string]$To,
|
|
[Parameter(Mandatory = $true)][string]$Username,
|
|
[Parameter(Mandatory = $true)][string]$Password,
|
|
[Parameter(Mandatory = $true)][string]$RouteName,
|
|
[Parameter(Mandatory = $true)][int]$TimeoutSeconds,
|
|
[Parameter(Mandatory = $true)][bool]$EnableSsl
|
|
)
|
|
|
|
$message = [System.Net.Mail.MailMessage]::new()
|
|
$smtp = [System.Net.Mail.SmtpClient]::new($Smarthost.Host, $Smarthost.Port)
|
|
|
|
try {
|
|
$message.From = [System.Net.Mail.MailAddress]::new($From)
|
|
foreach ($recipient in Split-Recipients -Value $To) {
|
|
$message.To.Add($recipient)
|
|
}
|
|
|
|
$message.Subject = "[ALERTING-LIVE-DRILL][$RouteName] $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')"
|
|
$message.Body = @"
|
|
This is a live alert delivery drill.
|
|
Route: $RouteName
|
|
Project: $projectRoot
|
|
GeneratedAt: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')
|
|
"@
|
|
|
|
$smtp.EnableSsl = $EnableSsl
|
|
$smtp.Timeout = $TimeoutSeconds * 1000
|
|
$smtp.DeliveryMethod = [System.Net.Mail.SmtpDeliveryMethod]::Network
|
|
$smtp.UseDefaultCredentials = $false
|
|
$smtp.Credentials = [System.Net.NetworkCredential]::new($Username, $Password)
|
|
$smtp.Send($message)
|
|
|
|
return [pscustomobject]@{
|
|
Route = $RouteName
|
|
RecipientMask = Mask-EmailList -Value $To
|
|
Accepted = $true
|
|
Error = ''
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
Route = $RouteName
|
|
RecipientMask = Mask-EmailList -Value $To
|
|
Accepted = $false
|
|
Error = $_.Exception.Message
|
|
}
|
|
} finally {
|
|
$message.Dispose()
|
|
$smtp.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-RedactedRenderedConfig {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$RenderedContent,
|
|
[Parameter(Mandatory = $true)][hashtable]$Values
|
|
)
|
|
|
|
$redacted = $RenderedContent
|
|
$replacementMap = @{
|
|
'ALERTMANAGER_DEFAULT_TO' = '***REDACTED_DEFAULT_TO***'
|
|
'ALERTMANAGER_CRITICAL_TO' = '***REDACTED_CRITICAL_TO***'
|
|
'ALERTMANAGER_WARNING_TO' = '***REDACTED_WARNING_TO***'
|
|
'ALERTMANAGER_FROM' = '***REDACTED_FROM***'
|
|
'ALERTMANAGER_SMARTHOST' = '***REDACTED_SMARTHOST***'
|
|
'ALERTMANAGER_AUTH_USERNAME' = '***REDACTED_AUTH_USERNAME***'
|
|
'ALERTMANAGER_AUTH_PASSWORD' = '***REDACTED_AUTH_PASSWORD***'
|
|
}
|
|
|
|
foreach ($entry in $replacementMap.GetEnumerator()) {
|
|
$value = [string]$Values[$entry.Key]
|
|
if ([string]::IsNullOrWhiteSpace($value)) {
|
|
continue
|
|
}
|
|
|
|
$redacted = [regex]::Replace($redacted, [regex]::Escape($value), [System.Text.RegularExpressions.MatchEvaluator]{ param($m) $entry.Value })
|
|
}
|
|
|
|
return $redacted
|
|
}
|
|
|
|
$savedEnvState = @()
|
|
$values = @{}
|
|
$missingVariables = @()
|
|
$placeholderFindings = @()
|
|
$renderSucceeded = $false
|
|
$tcpResult = [pscustomobject]@{ Succeeded = $false; Error = 'not-run' }
|
|
$sendResults = @()
|
|
$success = $false
|
|
$failureReason = ''
|
|
$smarthost = $null
|
|
$envSource = if ([string]::IsNullOrWhiteSpace($EnvFilePath)) { 'process environment' } else { $EnvFilePath }
|
|
|
|
try {
|
|
if (-not [string]::IsNullOrWhiteSpace($EnvFilePath)) {
|
|
if (-not (Test-Path $EnvFilePath)) {
|
|
throw "env file not found: $EnvFilePath"
|
|
}
|
|
|
|
$savedEnvState = Import-EnvFileToProcess -Path $EnvFilePath
|
|
}
|
|
|
|
$values = Get-ConfiguredValues -Names $requiredVariables
|
|
$missingVariables = @(
|
|
$requiredVariables |
|
|
Where-Object { [string]::IsNullOrWhiteSpace([string]$values[$_]) }
|
|
)
|
|
$placeholderFindings = Get-PlaceholderFindings -Values $values
|
|
|
|
if ($missingVariables.Count -gt 0) {
|
|
throw "missing required alertmanager variables: $($missingVariables -join ', ')"
|
|
}
|
|
|
|
if ($placeholderFindings.Count -gt 0) {
|
|
throw "placeholder or example values detected"
|
|
}
|
|
|
|
$smarthost = Parse-Smarthost -Value ([string]$values['ALERTMANAGER_SMARTHOST'])
|
|
|
|
& (Join-Path $PSScriptRoot 'render-alertmanager-config.ps1') `
|
|
-TemplatePath (Join-Path $projectRoot 'deployment\alertmanager\alertmanager.yml') `
|
|
-OutputPath $tempRenderedPath `
|
|
-EnvFilePath $EnvFilePath | Out-Null
|
|
|
|
$renderedContent = Get-Content $tempRenderedPath -Raw -Encoding UTF8
|
|
$redactedContent = Get-RedactedRenderedConfig -RenderedContent $renderedContent -Values $values
|
|
Set-Content -Path $sanitizedConfigPath -Value $redactedContent -Encoding UTF8
|
|
$renderSucceeded = $true
|
|
|
|
$tcpResult = Test-TcpConnectivity -Host $smarthost.Host -Port $smarthost.Port -TimeoutSeconds $TimeoutSeconds
|
|
if (-not $tcpResult.Succeeded) {
|
|
throw "smtp tcp connectivity failed: $($tcpResult.Error)"
|
|
}
|
|
|
|
$routes = @(
|
|
[pscustomobject]@{ Name = 'default'; To = [string]$values['ALERTMANAGER_DEFAULT_TO'] }
|
|
[pscustomobject]@{ Name = 'critical-alerts'; To = [string]$values['ALERTMANAGER_CRITICAL_TO'] }
|
|
[pscustomobject]@{ Name = 'warning-alerts'; To = [string]$values['ALERTMANAGER_WARNING_TO'] }
|
|
)
|
|
|
|
foreach ($route in $routes) {
|
|
$sendResults += Send-SmtpMessage `
|
|
-Smarthost $smarthost `
|
|
-From ([string]$values['ALERTMANAGER_FROM']) `
|
|
-To $route.To `
|
|
-Username ([string]$values['ALERTMANAGER_AUTH_USERNAME']) `
|
|
-Password ([string]$values['ALERTMANAGER_AUTH_PASSWORD']) `
|
|
-RouteName $route.Name `
|
|
-TimeoutSeconds $TimeoutSeconds `
|
|
-EnableSsl (-not $DisableSsl.IsPresent)
|
|
}
|
|
|
|
$failedRoutes = @($sendResults | Where-Object { -not $_.Accepted })
|
|
if ($failedRoutes.Count -gt 0) {
|
|
throw "smtp send failed for route(s): $($failedRoutes.Route -join ', ')"
|
|
}
|
|
|
|
$success = $true
|
|
} catch {
|
|
$failureReason = $_.Exception.Message
|
|
} finally {
|
|
if (Test-Path $tempRenderedPath) {
|
|
Remove-Item $tempRenderedPath -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
if ($savedEnvState.Count -gt 0) {
|
|
Restore-ProcessEnv -SavedState $savedEnvState
|
|
}
|
|
|
|
$reportLines = @(
|
|
'# Alertmanager Live Delivery Drill',
|
|
'',
|
|
"- Generated at: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')",
|
|
"- Template file: $(Join-Path $projectRoot 'deployment\alertmanager\alertmanager.yml')",
|
|
"- Env source: $envSource",
|
|
"- Redacted rendered config: $(if (Test-Path $sanitizedConfigPath) { $sanitizedConfigPath } else { 'not-generated' })",
|
|
'',
|
|
'## Strict Preconditions',
|
|
'',
|
|
"- Required variables present: $($missingVariables.Count -eq 0)",
|
|
"- Placeholder/example-value findings: $(if ($placeholderFindings.Count -gt 0) { $placeholderFindings -join '; ' } else { 'none' })",
|
|
"- Render path succeeded: $renderSucceeded",
|
|
'',
|
|
'## Delivery Attempt',
|
|
'',
|
|
"- SMTP host: $(if ($smarthost) { (Mask-Host -Value $smarthost.Host) } else { 'unparsed' })",
|
|
"- SMTP port: $(if ($smarthost) { $smarthost.Port } else { 'unparsed' })",
|
|
"- TLS enabled: $(-not $DisableSsl.IsPresent)",
|
|
"- TCP connectivity succeeded: $($tcpResult.Succeeded)",
|
|
"- TCP connectivity error: $(if ($tcpResult.Error) { $tcpResult.Error } else { 'none' })",
|
|
''
|
|
)
|
|
|
|
if ($sendResults.Count -gt 0) {
|
|
$reportLines += '## Route Results'
|
|
$reportLines += ''
|
|
foreach ($result in $sendResults) {
|
|
$reportLines += "- Route $($result.Route): accepted=$($result.Accepted), recipients=$($result.RecipientMask), error=$(if ([string]::IsNullOrWhiteSpace($result.Error)) { 'none' } else { $result.Error })"
|
|
}
|
|
$reportLines += ''
|
|
}
|
|
|
|
$reportLines += '## Conclusion'
|
|
$reportLines += ''
|
|
$reportLines += "- Live external delivery closed: $success"
|
|
$reportLines += "- Failure reason: $(if ([string]::IsNullOrWhiteSpace($failureReason)) { 'none' } else { $failureReason })"
|
|
$reportLines += '- This drill fails closed on unresolved placeholders, example domains, and placeholder secrets.'
|
|
$reportLines += '- The evidence intentionally stores only redacted config output and masked recipient information.'
|
|
$reportLines += '- A successful run proves real secret injection plus SMTP server acceptance for the configured on-call routes; it does not by itself prove downstream human acknowledgment.'
|
|
$reportLines += ''
|
|
|
|
Set-Content -Path $reportPath -Value ($reportLines -join [Environment]::NewLine) -Encoding UTF8
|
|
Get-Content $reportPath
|
|
}
|
|
|
|
if (-not $success) {
|
|
throw "alertmanager live delivery drill failed: $failureReason"
|
|
}
|