77 lines
3.0 KiB
Python
77 lines
3.0 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Check what the carrier did with the messages we handed over.
|
||
|
|
|
||
|
|
Run on a schedule. Exits non-zero when anything was accepted and then not
|
||
|
|
delivered, so a cron mailer or a supervisor surfaces it — the point being that
|
||
|
|
somebody hears about it. Forty-eight consecutive failures went unnoticed for
|
||
|
|
seven months because nothing ever asked this question.
|
||
|
|
|
||
|
|
python3 scripts/reconcile.py # last 24 hours
|
||
|
|
python3 scripts/reconcile.py --hours 168 # last week
|
||
|
|
python3 scripts/reconcile.py --email [email protected] # and post the report
|
||
|
|
|
||
|
|
Reporting by email goes through this relay's own send path, so the alert
|
||
|
|
travels the one channel that is known to work. If SMS is the thing that is
|
||
|
|
broken, an SMS alert about it would be the last thing to arrive.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
|
||
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
|
|
||
|
|
from app.config import Settings # noqa: E402
|
||
|
|
from app.reconcile import ( # noqa: E402
|
||
|
|
describe, describe_email, email_bounces, fetch, reconcile, window_start,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument("--hours", type=int, default=24, help="how far back to look")
|
||
|
|
parser.add_argument("--email", help="send the report to this address as well")
|
||
|
|
parser.add_argument("--quiet-when-clean", action="store_true",
|
||
|
|
help="print nothing when every message was delivered")
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
settings = Settings()
|
||
|
|
if not (settings.twilio_sid and settings.twilio_token):
|
||
|
|
print("no Twilio credentials configured; nothing to reconcile", file=sys.stderr)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
messages = fetch(account_sid=settings.twilio_sid, auth_token=settings.twilio_token,
|
||
|
|
since=window_start(args.hours), timeout=settings.timeout)
|
||
|
|
report = reconcile(messages)
|
||
|
|
text = describe(report)
|
||
|
|
|
||
|
|
# Email is asked about too, and says plainly when it cannot be answered —
|
||
|
|
# a check that silently skips the channel it cannot see is how a gap
|
||
|
|
# becomes invisible.
|
||
|
|
email = email_bounces(api_key=settings.sendgrid_key,
|
||
|
|
since=window_start(args.hours), timeout=settings.timeout)
|
||
|
|
text = text + "\n\n" + describe_email(email)
|
||
|
|
|
||
|
|
if report.ok and email.checked and not email.bounces and args.quiet_when_clean:
|
||
|
|
return 0
|
||
|
|
print(text)
|
||
|
|
|
||
|
|
if args.email and report.failed:
|
||
|
|
# Only on failure: a daily "everything is fine" email is a thing people
|
||
|
|
# filter, and then the one that matters is filtered too.
|
||
|
|
from app.providers import send_email
|
||
|
|
|
||
|
|
outcome = send_email(
|
||
|
|
to=args.email, subject=f"SMS not delivered: {len(report.failed)} in the last {args.hours}h",
|
||
|
|
body=text, api_key=settings.sendgrid_key, sender=settings.email_from,
|
||
|
|
timeout=settings.timeout)
|
||
|
|
print(f"report emailed: provider status {outcome.status}", file=sys.stderr)
|
||
|
|
|
||
|
|
return 1 if report.failed else 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|