1515 lines
50 KiB
Python
1515 lines
50 KiB
Python
r"""
|
|
Parse postfix log messages.
|
|
|
|
The parser (:func:`parse_entry`) uses regular expressions to produce a dict.
|
|
This dict is used by :func:`extract_delivery` to extract relevant mail
|
|
delivery information.
|
|
|
|
If VERP id parsing is to be used, call :func:`init` before using
|
|
:func:`parse`.
|
|
|
|
Comments like 'smtpd/smtpd.c:1663-1664' refer to a postfix-3.4.7 source file
|
|
and line numbers within that file.
|
|
|
|
We assume that verbose logging is off, which is the default (util/msg.c:177).
|
|
We do not parse verbose messages (variable "msg_verbose", usually).
|
|
Useful command for searching postfix sources (in subdirs global, smtpd, qmgr,
|
|
...):
|
|
|
|
rg 'msg_info\("%s:' -B2 -n *.c
|
|
|
|
We only parse messages of level info and some of level warning
|
|
(see util/msg.c:67-71, also cf. postlog/postlog.c:140-150);
|
|
messages of levels 'error', 'fatal', 'panic' are discarded.
|
|
|
|
We only parse messages with queue_id (considering "NOQUEUE" as
|
|
queue_id == None).
|
|
|
|
Coverage of postfix daemon components:
|
|
|
|
* smtpd: yes (includes submission)
|
|
* trivial-rewrite: yes (no relevant messages of level msg_info)
|
|
* cleanup: yes
|
|
* qmgr: yes
|
|
* smtp, lmtp: mostly
|
|
* bounce: mostly
|
|
* virtual: mostly
|
|
* error: partially
|
|
|
|
These components are not covered:
|
|
|
|
* anvil
|
|
* discard
|
|
* dnsblog
|
|
* flush
|
|
* local
|
|
* oqmgr
|
|
* pickup
|
|
* pipe
|
|
* postlogd
|
|
* postscreen
|
|
* proxymap
|
|
* qmqpd
|
|
* scache
|
|
* showq
|
|
* tlsmgr
|
|
* tlsproxy
|
|
* verify
|
|
|
|
Note: In particular, local delivery is not supported!
|
|
|
|
For lmtp we try to extract a dovecot_id from the delivery status text,
|
|
see :func:`_get_delivery_status`.
|
|
"""
|
|
|
|
import re
|
|
from pprint import pprint
|
|
from traceback import format_exc
|
|
from typing import List, Optional, Tuple, Union
|
|
|
|
|
|
ignore_identifiers = (
|
|
'postfix/master',
|
|
'postfix/postfix-script',
|
|
'configure-instance.sh',
|
|
'postmulti',
|
|
)
|
|
"""
|
|
Syslog identifiers to ignore.
|
|
"""
|
|
|
|
|
|
where = (
|
|
'CONNECT',
|
|
'DATA content',
|
|
'BDAT content',
|
|
'END-OF-MESSAGE',
|
|
'HELO',
|
|
'EHLO',
|
|
'STARTTLS',
|
|
'AUTH',
|
|
'MAIL',
|
|
'RCPT',
|
|
'DATA',
|
|
'BDAT',
|
|
'RSET',
|
|
'NOOP',
|
|
'VRFY',
|
|
'ETRN',
|
|
'QUIT',
|
|
'XCLIENT',
|
|
'XFORWARD',
|
|
'UNKNOWN',
|
|
)
|
|
"""
|
|
Possible smtpd "where" values from smtpd/smtpd.c:235-260.
|
|
"""
|
|
|
|
|
|
smtpd_whatsup_actions = {
|
|
'reject': 'reject',
|
|
'hangup': 'reject',
|
|
'info': None,
|
|
'warn': None,
|
|
'filter': None,
|
|
'hold': 'hold',
|
|
'delay': 'delay',
|
|
'discard': 'discard',
|
|
'redirect': 'redirect',
|
|
'bcc': None,
|
|
'permit': None,
|
|
'reject_warning': 'reject',
|
|
}
|
|
"""
|
|
Keys are from `rg log_whatsup smtpd/*.c` and smtpd/s,tpd_check.c:998,1038.
|
|
|
|
Map the possible smtpd whatsup value to our action.
|
|
"""
|
|
|
|
|
|
cleanup_actions = {
|
|
'reject': 'reject',
|
|
'warning': None,
|
|
'info': None,
|
|
'filter': None,
|
|
'pass': None,
|
|
'discard': 'discard',
|
|
'hold': 'hold',
|
|
'delay': 'delay',
|
|
'prepend': None,
|
|
'replace': None,
|
|
'redirect': 'redirect',
|
|
'bcc': None,
|
|
'strip': None,
|
|
}
|
|
"""
|
|
Possible cleanup "actions" and a mapping to our action.
|
|
"""
|
|
|
|
|
|
smtp_hbc_actions = ['warning', 'info', 'replace', 'prepend', 'strip']
|
|
"""
|
|
SMTP header body check actions.
|
|
"""
|
|
|
|
|
|
cleanup_contexts = ('header', 'body', 'content')
|
|
|
|
|
|
mime_error_texts = [
|
|
'MIME nesting exceeds safety limit',
|
|
'message header length exceeds safety limit',
|
|
'improper use of 8-bit data in message header',
|
|
'improper use of 8-bit data in message body',
|
|
'invalid message/* or multipart/* encoding domain',
|
|
]
|
|
"""
|
|
MIME error texts in cleanup.
|
|
"""
|
|
|
|
|
|
# rfc3464_actions = ['failed', 'delayed', 'delivered', 'relayed', 'expanded']
|
|
|
|
|
|
address_pattern = r'([^">]*|"([^ "\\]|\\[^ ])*"@[^>]*)'
|
|
"""
|
|
Email address pattern.
|
|
|
|
Either match any number of characters not containing '"' or '>', or
|
|
match a local part followed by '@' and a domain part, where the
|
|
domain part is arbitrary, but must not contain '>', and the local
|
|
part begins and ends with a '"' and contains 1) any char except
|
|
space, tab, '"', r'\' or 2) any char except tab prepended by r'\'.
|
|
|
|
Note:
|
|
|
|
* email addresses are by default logged in 'external' format by Postfix >=3.5:
|
|
http://www.postfix.org/postconf.5.html#info_log_address_format
|
|
* https://stackoverflow.com/questions/201323/how-to-validate-an-email-address-using-a-regular-expression
|
|
"""
|
|
|
|
|
|
regexp_patterns = {
|
|
'queue_id_short': r'([0-9A-F]{12})',
|
|
'queue_id_long': r'([0-9BCDFGHIJKLMNPQRSTVWXYZbcdfghijklmnpqrstvwxyz]{10,15}z[0-9BCDFGHIJKLMNPQRSTVWXYZbcdfghijklmnpqrstvwxy]{5,10})',
|
|
'from': 'from=<' + address_pattern + '>',
|
|
'to': 'to=<' + address_pattern + '>',
|
|
'orig_to': 'orig_to=<' + address_pattern + '>',
|
|
'message-id': r'message-id=<([^>]*)>',
|
|
'nrcpt': r'nrcpt=([0-9]+)',
|
|
'relay': r'relay=(none|local|virtual|([^ ]+)\[([^\]]+)\](:([0-9]+))?)', # matches 5 values
|
|
'delay': r'delay=([0-9\.]+)',
|
|
'delays': r'delays=([0-9\./]+)',
|
|
'dsn': r'dsn=([0-9\.]+)',
|
|
'proto': r'proto=(SMTP|ESMTP)',
|
|
'helo': r'helo=<([^>]*)>',
|
|
'host_ip': r'([^ ]+)\[([^\]]+)\]', # matches 2 values
|
|
'none': r'none',
|
|
'dovecot_id': r'([A-za-z0-9/\+]{22}) Saved',
|
|
'sasl_username': r'sasl_username=<?(.*)>?',
|
|
'sasl_method': r'sasl_method=(.*)',
|
|
'size': r'size=([0-9]+)',
|
|
'orig_client': r'orig_client=(.*)',
|
|
'orig_queue_id': r'orig_queue_id=(.*)',
|
|
'sasl_sender': r'sasl_sender=(.*)',
|
|
'cleanup_context': '(' + '|'.join(cleanup_contexts) + ')',
|
|
}
|
|
"""
|
|
regexp patterns, usually for matching one expression.
|
|
"""
|
|
|
|
|
|
cleanup_milter_apply_events = (
|
|
'END-OF-MESSAGE',
|
|
'CONNECT',
|
|
'EHLO',
|
|
'MAIL',
|
|
'RCPT',
|
|
'DATA',
|
|
)
|
|
"""
|
|
cleanup milter stages.
|
|
"""
|
|
|
|
|
|
regexps = {
|
|
'cleanup_optional_text': re.compile(' helo=<[^>]*>: '),
|
|
'failed_mail': re.compile(
|
|
r'^([^\[]+)\[([^\]]+)\]: (.*); proto=[^ ]+ helo=<([^>]+)>$'
|
|
),
|
|
'failed_rcpt': re.compile(
|
|
r'^([^\[]+)\[([^\]]+)\]: (.*); from=<([^>]*)> to=<([^>]+)> proto=[^ ]+ helo=<([^>]+)>$'
|
|
),
|
|
}
|
|
"""
|
|
Special regular expressions for matching expressions which are harder to parse.
|
|
"""
|
|
|
|
|
|
for label, pattern in regexp_patterns.items():
|
|
regexps['l_' + label] = re.compile(r'(^' + pattern + r'(: |, | |$))')
|
|
regexps['r_' + label] = re.compile(r'( ?' + pattern + r'$)')
|
|
|
|
|
|
def _strip_queue_id(msg, res, pos='l', target_name='queue_id'):
|
|
"""
|
|
Strip a postfix queue_id at the left or right end of *msg*.
|
|
|
|
The queue_id can either be a short one, or a long one.
|
|
If none is matched, return *msg*.
|
|
"""
|
|
m = regexps[pos + '_queue_id_short'].search(msg)
|
|
if not m:
|
|
m = regexps[pos + '_queue_id_long'].search(msg)
|
|
if not m:
|
|
return msg
|
|
res[target_name] = m.group(2)
|
|
l_ = len(m.group(1))
|
|
return msg[l_:] if pos == 'l' else msg[:-l_]
|
|
|
|
|
|
def _strip_pattern(msg, res, pattern_name, pos='l', target_names=None):
|
|
"""
|
|
Strip a pattern at the left/right end of *msg* and store fields.
|
|
|
|
Matching at the left (right) end is chosen be pos='l' (pos='r').
|
|
*pattern_name* is a key from regexp_patterns.
|
|
If target_names is set, it must be an iterable. Each name in it
|
|
is used as a key in *res* and the values are set to values matched
|
|
by the pattern. Most patterns in regexp_patterns only match one
|
|
value; in this case the by default (target_names=None) the
|
|
*pattern_name* will be used as key in *res*.
|
|
"""
|
|
m = regexps[pos + '_' + pattern_name].search(msg)
|
|
if m:
|
|
if target_names is None:
|
|
target_names = (pattern_name,)
|
|
for ind, target_name in enumerate(target_names):
|
|
res[target_name] = m.group(2 + ind)
|
|
l_ = len(m.group(1))
|
|
msg = msg[l_:] if pos == 'l' else msg[:-l_]
|
|
return msg
|
|
|
|
|
|
def _rstrip_smtpd_whatsup(msg, res):
|
|
"""
|
|
Strip and store smtpd_whatsup fields at the right end of *msg*.
|
|
|
|
Return the msg part in front of them.
|
|
"""
|
|
# from=<{sender}> to=<{recipient}> proto=ESMTP helo=<{helo}> sasl_username=<{sasl_username}>
|
|
for field_name in ('sasl_username', 'helo', 'proto', 'to', 'from'):
|
|
msg = _strip_pattern(msg, res, field_name, pos='r')
|
|
return msg
|
|
|
|
|
|
def _strip_host_ip(msg, res, pos='l', allow_none=False):
|
|
"""
|
|
Strip a hostname followed by an IP address in brackets from *msg*.
|
|
|
|
*pos* determines whether the pattern is matched on the left or
|
|
right hand side of *msg*.
|
|
|
|
The hostname is put into res['host'], the IP address into res['ip'].
|
|
If the hostname equals "unknown", it is set to None.
|
|
|
|
If allow_none evaluates to True and msg == 'none', this results in
|
|
res['host'] = None and res['ip'] = None.
|
|
"""
|
|
if allow_none:
|
|
msg_ = _strip_pattern(msg, res, 'none', pos=pos)
|
|
if msg_ != msg: # "none" was matched
|
|
res['host'] = None
|
|
res['ip'] = None
|
|
return msg_
|
|
msg = _strip_pattern(
|
|
msg, res, 'host_ip', pos=pos, target_names=('host', 'ip')
|
|
)
|
|
if res.get('host') == 'unknown':
|
|
res['host'] = None
|
|
return msg
|
|
|
|
|
|
def _strip_relay(msg, res, pos='l'):
|
|
"""
|
|
Strip a relay pattern from *msg*.
|
|
|
|
If "relay=none" or "relay=local" or "relay=virtual" we
|
|
set host,destination,port in res to None.
|
|
The destination can be an IP address or text
|
|
(e.g., "private/dovecot-lmtp").
|
|
"""
|
|
msg = _strip_pattern(
|
|
msg,
|
|
res,
|
|
'relay',
|
|
pos=pos,
|
|
target_names=('relay_full', 'host', 'destination', 'port_', 'port'),
|
|
)
|
|
if res.get('relay_full') in ('none', 'local', 'virtual'):
|
|
res['relay'] = res.get('relay_full')
|
|
res['host'] = None
|
|
res['destination'] = None
|
|
res['port'] = None
|
|
else:
|
|
res['relay'] = 'external'
|
|
if res['destination'] == 'private/dovecot-lmtp':
|
|
res['relay'] = 'lmtp'
|
|
return msg
|
|
|
|
|
|
def _lstrip_where(msg, res):
|
|
"""
|
|
Strip and store 'where' string at the left end of *msg*.
|
|
|
|
Return *msg*, where the 'where' string was strippped, if found.
|
|
See global variable 'where'.
|
|
"""
|
|
for where_ in where:
|
|
if msg.startswith(where_):
|
|
res['where'] = where_
|
|
return msg[len(where_):]
|
|
return msg
|
|
|
|
|
|
def _strip(text, part, pos):
|
|
"""
|
|
Strip *part* from *text* on the left (pos='l') or right (pos='r') hand side.
|
|
"""
|
|
if pos == 'l':
|
|
if text.startswith(part):
|
|
return text[len(part):]
|
|
if pos == 'r':
|
|
if text.endswith(part):
|
|
return text[:-len(part)]
|
|
return text
|
|
|
|
|
|
def _parse_log_adhoc(msg, res):
|
|
"""
|
|
Parse a log message formatted in global/log_adhoc.c:82-214 (log_adhoc).
|
|
|
|
The message has one of these two formats:
|
|
|
|
to=<{to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
to=<{to}>, orig_to=<{orig_to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
|
|
The resulting action will be 'delivery_status'.
|
|
"""
|
|
if msg.startswith('to=<'):
|
|
msg_ = _strip_pattern(msg, res, 'to', pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'orig_to', pos='l')
|
|
msg_ = _strip_relay(msg_, res, pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'delay', pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'delays', pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'dsn', pos='l')
|
|
_get_delivery_status(msg_, res)
|
|
if 'delivery_status' not in res:
|
|
res['parsed'] = False
|
|
return msg
|
|
|
|
|
|
def _get_delivery_status(msg, res):
|
|
"""
|
|
Extract the delivery status from the beginning of *msg* into *res*.
|
|
|
|
A delivery status looks like this:
|
|
|
|
status={status} ({text})
|
|
|
|
Here {status} is a word (has no spaces).
|
|
|
|
The result is put into res['delivery_status'] and res['delivery_status_text'].
|
|
"""
|
|
if msg.startswith('status='):
|
|
res['action'] = 'delivery_status'
|
|
if ' ' in msg:
|
|
status, detail = msg.split(' ', 1)
|
|
res['delivery_status'] = status[7:]
|
|
res['delivery_status_text'] = detail.lstrip('(').rstrip(')')
|
|
# also try to extract a dovecot_id; example:
|
|
# 250 2.0.0 <user@example.com> 9y4WEP+qV11uBgAAZU03Dg Saved
|
|
_strip_pattern(
|
|
res['delivery_status_text'], res, 'dovecot_id', pos='r'
|
|
)
|
|
else: # can this happen at all?
|
|
res['delivery_status'] = msg[7:]
|
|
return msg
|
|
|
|
|
|
def init_parser(verp_marker: Optional[str] = None):
|
|
"""
|
|
Init the module.
|
|
|
|
*verp_marker* is the VERP marker.
|
|
"""
|
|
global regexps
|
|
if verp_marker:
|
|
regexps['verp_id'] = re.compile(
|
|
r'(.*\+.*)' + verp_marker + r'-(\d+)(@[^@]+$)'
|
|
)
|
|
|
|
|
|
def parse_entry(
|
|
msg_details: dict, debug: Union[bool, List[str]] = False
|
|
) -> Optional[dict]:
|
|
"""
|
|
Parse a log message returning a dict.
|
|
|
|
*msg_details* is assumed to be a dict with these keys:
|
|
|
|
* 'identifier' (syslog identifier),
|
|
* 'pid' (process id),
|
|
* 'message' (message text)
|
|
|
|
The syslog identifier and process id are copied to the resulting dict.
|
|
"""
|
|
identifier = msg_details['identifier']
|
|
if identifier in ignore_identifiers:
|
|
return None
|
|
pid = msg_details['pid']
|
|
message = msg_details['message'].strip()
|
|
# postfix component
|
|
component = (
|
|
identifier[8:] if identifier.startswith('postfix/') else identifier
|
|
)
|
|
res = {'comp': component, 'pid': pid}
|
|
# do we have a postfix queue identifer?
|
|
if message.startswith('NOQUEUE: '):
|
|
res['queue_id'] = None
|
|
msg_ = message[9:]
|
|
else:
|
|
# do not put key 'queue_id' into res, if not found
|
|
msg_ = _strip_queue_id(message, res)
|
|
try:
|
|
if 'queue_id' in res:
|
|
_parse_branch(component, msg_, res)
|
|
except Exception:
|
|
res['parsed'] = False
|
|
print('PARSING FAILED:', message)
|
|
print(format_exc())
|
|
res['parsed'] = not res.get('parsed') is False
|
|
if debug:
|
|
if not res['parsed']:
|
|
print('-' * 20, message)
|
|
print(component)
|
|
elif 'queue_id' in res and res.get('action') == 'ignore':
|
|
print('I' * 20, message)
|
|
print(component)
|
|
else:
|
|
print(message)
|
|
pprint(res)
|
|
print('_' * 100)
|
|
return res
|
|
|
|
|
|
def _parse_branch(comp: str, msg: str, res: dict) -> None:
|
|
"""
|
|
Parse a log message string *msg*, adding results to dict *res*.
|
|
|
|
Depending on the component *comp* we branch to functions
|
|
named _parse_{comp}.
|
|
|
|
Add parsing results to dict *res*. Always add key 'action'.
|
|
Try to parse every syntactical element.
|
|
Note: We parse what we can. Assessment of parsing results relevant
|
|
for delivery is done in :func:`extract_delivery`.
|
|
"""
|
|
if (
|
|
msg.startswith('warning: ')
|
|
or msg.startswith('error: ')
|
|
or msg.startswith('fatal: ')
|
|
or msg.startswith('panic: ')
|
|
or msg.startswith('using backwards-compatible default setting')
|
|
):
|
|
res['action'] = 'ignore'
|
|
return
|
|
if comp == 'smtpd' or comp.endswith(
|
|
'/smtpd'
|
|
): # includes 'submission/smtpd'
|
|
_parse_smtpd(msg, res)
|
|
res['submission'] = comp.startswith('submission')
|
|
elif comp == 'qmgr':
|
|
_parse_qmgr(msg, res)
|
|
elif comp == 'cleanup':
|
|
_parse_cleanup(msg, res)
|
|
elif comp == 'trivial-rewrite':
|
|
_parse_trivial_rewrite(msg, res)
|
|
elif comp in ('smtp', 'lmtp') or comp.endswith(
|
|
'/smtp'
|
|
): # includes 'relay/smtp'
|
|
_parse_smtp(msg, res)
|
|
res['smtp_relay'] = comp.startswith('relay')
|
|
elif comp == 'bounce':
|
|
_parse_bounce(msg, res)
|
|
elif comp == 'virtual':
|
|
_parse_virtual(msg, res)
|
|
elif comp == 'error':
|
|
_parse_error(msg, res)
|
|
else:
|
|
res['parsed'] = False
|
|
|
|
# extract a possible verp_id from orig_to
|
|
if 'orig_to' in res:
|
|
res['verp_id'], res['orig_to'] = _parse_verp_id(res['orig_to'])
|
|
elif 'from' in res:
|
|
res['verp_id'], res['from'] = _parse_verp_id(res['from'])
|
|
|
|
|
|
def _parse_verp_id(email):
|
|
"""
|
|
Return th VERP id and the original email.
|
|
"""
|
|
if 'verp_id' in regexps:
|
|
m = regexps['verp_id'].match(email)
|
|
if m:
|
|
verp_id = m.group(2)
|
|
orig_email = m.group(1).rstrip('+') + m.group(3)
|
|
return verp_id, orig_email
|
|
return None, email
|
|
|
|
|
|
def _parse_smtpd(msg, res):
|
|
"""
|
|
Parse log messages of the smtpd component, including submission.
|
|
"""
|
|
# smtpd/smtpd.c:2229-2246 smtpd_sasl_auth_cmd_wrapper
|
|
# client={hostname_or_unknown}[{ip_address}]
|
|
# client={hostname_or_unknown}[{ip_address}], sasl_method={sasl_method}, sasl_username={sasl_username}, sasl_sender={sasl_sender}, orig_queue_id={orig_queue_id}, orig_client={orig_client}
|
|
# (sasl_* and orig_* are individually optional)
|
|
if msg.startswith('client='):
|
|
msg_ = _strip_host_ip(msg[7:], res, pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'orig_client', pos='r')
|
|
msg_ = _strip_pattern(msg_, res, 'orig_queue_id', pos='r')
|
|
msg_ = _strip_pattern(msg_, res, 'sasl_sender', pos='r')
|
|
msg_ = _strip_pattern(msg_, res, 'sasl_username', pos='r')
|
|
msg_ = _strip_pattern(msg_, res, 'sasl_method', pos='r')
|
|
res['action'] = 'connect'
|
|
|
|
# smtpd/smtpd_check.c:949-967 log_whatsup
|
|
# {smtpd_whatsup}: {where} from {hostname_or_unknown}[{ip_address}]: {error}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>
|
|
# smtpd/smtpd.c:5411-5414 smtpd_proto
|
|
# reject: {where} from {hostname_or_unknown}[{ip_address}]: 421 4.3.0 {myhostname} Server local data error
|
|
# reject: {where} from {hostname_or_unknown}[{ip_address}]: {error}
|
|
elif ': ' in msg and msg.split(': ', 1)[0] in smtpd_whatsup_actions.keys():
|
|
smtpd_whatsup, msg_ = msg.split(': ', 1)
|
|
res['smtpd_whatsup'] = smtpd_whatsup
|
|
if smtpd_whatsup_actions.get(smtpd_whatsup):
|
|
res['action'] = smtpd_whatsup_actions.get(smtpd_whatsup)
|
|
else:
|
|
res['action'] = 'ignore'
|
|
msg_ = _lstrip_where(msg_, res)
|
|
msg_ = _strip(msg_, ' from ', 'l')
|
|
msg_ = _strip_host_ip(msg_, res, pos='l')
|
|
msg_ = _strip(msg_, ': ', 'l')
|
|
msg_ = _rstrip_smtpd_whatsup(msg_, res)
|
|
msg_ = _strip(msg_, ';', 'r')
|
|
res['error'] = msg_
|
|
|
|
# smtpd/smtpd.c:1663-1664 check_milter_reply
|
|
# {milter_action}: {where} from {hostname_or_unknown}[{ip_address}]: {error}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>
|
|
elif msg.startswith('milter-hold: '):
|
|
msg_ = _strip(_rstrip_smtpd_whatsup(msg[13:], res), ';', 'r')
|
|
msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l')
|
|
msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l')
|
|
res['error'] = msg_
|
|
res['action'] = 'hold'
|
|
elif msg.startswith('milter-discard: '):
|
|
msg_ = _strip(_rstrip_smtpd_whatsup(msg[16:], res), ';', 'r')
|
|
msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l')
|
|
msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l')
|
|
res['error'] = msg_
|
|
res['action'] = 'discard'
|
|
elif msg.startswith('milter-reject: '):
|
|
msg_ = _strip(_rstrip_smtpd_whatsup(msg[15:], res), ';', 'r')
|
|
msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l')
|
|
msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l')
|
|
res['error'] = msg_
|
|
res['action'] = 'reject'
|
|
|
|
# smtpd/smtpd.c:5099-5102 smtpd_start_tls
|
|
# abort: TLS from {hostname_or_unknown}[{ip_address}]: {error}
|
|
elif (
|
|
'queue_id' in res
|
|
and res['queue_id'] is None
|
|
and msg.startswith('abort: TLS from ')
|
|
):
|
|
msg_ = msg[16:]
|
|
msg_ = _strip_host_ip(msg_, res, pos='l')
|
|
msg_ = _strip(msg_, ': ', 'l')
|
|
res['error'] = msg_
|
|
res['action'] = 'reject'
|
|
|
|
# smtpd/smtpd.c:1850 ehlo_cmd
|
|
# discarding EHLO keywords: {keywords}
|
|
elif msg.startswith('discarding EHLO keywords: '):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtpd/smtpd.c:5624 smtpd_proto
|
|
# replacing command {command1} with {command2}
|
|
# without queue_id
|
|
# -> ignore
|
|
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_qmgr(msg, res):
|
|
"""
|
|
Parse log messages of the qmgr component.
|
|
"""
|
|
# qmgr/qmgr_active.c:441 qmgr_active_done_25_generic
|
|
# from=<{from}>, status=expired, returned to sender
|
|
if msg.startswith('from=<') and msg.endswith(
|
|
', status=expired, returned to sender'
|
|
):
|
|
res['from'] = msg[6:-36].rstrip('>')
|
|
res['action'] = 'expired'
|
|
|
|
# qmgr/qmgr_active.c:520 qmgr_active_done_3_generic
|
|
elif msg == 'removed':
|
|
res['action'] = 'removed'
|
|
|
|
# qmgr/qmgr.c:680 pre_accept
|
|
# %s: %s feedback type %d value at %d: %g
|
|
# without queue_id
|
|
# -> ignore
|
|
|
|
# qmgr/qmgr_feedback.c:170
|
|
# "%s: %s feedback type %d value at %d: %g"
|
|
# without queue_id
|
|
# -> ignore
|
|
|
|
# qmgr/qmgr_message.c:642-644 qmgr_message_read
|
|
# global/opened.c:64,70,76,84-86 opened,vopened
|
|
# from=<{from}>, size={size}, nrcpt={nrcpt} (queue {queue_name})
|
|
elif msg.startswith('from=<') and ', nrcpt=' in msg:
|
|
msg_ = _strip_pattern(msg, res, 'from', pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'size', pos='l')
|
|
msg_ = _strip_pattern(msg_, res, 'nrcpt', pos='l')
|
|
msg_ = _strip(msg_, '(queue ', 'l').rstrip(')')
|
|
if msg_:
|
|
res['queue_name'] = msg_
|
|
res['action'] = 'queued'
|
|
|
|
# qmgr/qmgr_message.c:1493 qmgr_message_alloc - create in-core message structure
|
|
elif msg == 'skipped, still being delivered':
|
|
res['action'] = 'skipped'
|
|
|
|
# qmgr/qmgr_queue.c:124-132
|
|
# "%s: feedback %g"
|
|
# "%s: queue %s: limit %d window %d success %g failure %g fail_cohorts %g"
|
|
# without queue_id
|
|
# -> ignore
|
|
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_cleanup(msg, res):
|
|
"""
|
|
Parse log messages of the cleanup component.
|
|
"""
|
|
if msg.startswith('info: '):
|
|
msg_ = msg[6:]
|
|
if msg_.lower().startswith('header subject: '):
|
|
msg_ = _rstrip_smtpd_whatsup(msg_[16:], res).rstrip(';')
|
|
res['subject'] = _strip(
|
|
_strip_host_ip(msg_, res, pos='r'), ' from', 'r'
|
|
)
|
|
res['action'] = 'subject'
|
|
else:
|
|
res['action'] = 'ignore'
|
|
|
|
# cleanup/cleanup.c:584
|
|
elif msg.startswith('table ') and msg.endswith(
|
|
' has changed -- restarting'
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# cleanup/cleanup_masquerade.c:111
|
|
# "%s: %s map lookup problem -- "
|
|
# -> ignore
|
|
|
|
# cleanup/cleanup_message.c:1025 cleanup_mime_error_callback
|
|
# reject: mime-error {mime_error}: {mime_error_detail} from {unknown_or_host_ip}; from=<{from}> to=<{to}>
|
|
elif msg.startswith('reject: mime-error '):
|
|
res['action'] = 'reject'
|
|
msg_ = msg[19:]
|
|
for mime_error_text in mime_error_texts:
|
|
if msg_.startswith(mime_error_text):
|
|
res['mime_error'] = mime_error_text
|
|
msg_ = msg_[len(mime_error_text):]
|
|
for field_name in ('to', 'from'):
|
|
msg_ = _strip_pattern(msg_, res, field_name, pos='r')
|
|
if msg_.endswith(' unknown'):
|
|
msg_ = msg_[:-8]
|
|
else:
|
|
msg_ = _strip_host_ip(msg_, res, pos='r')
|
|
msg_ = _strip(msg_, ' from', 'r')
|
|
msg_ = _strip(msg_, ': ', 'l')
|
|
if 'mime_error' in res:
|
|
res['mime_error_detail'] = msg_
|
|
|
|
# cleanup/cleanup_message.c:255-276 cleanup_act_log
|
|
# {cleanup_action}: {class} {content} from {attr}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>: {optional_text}
|
|
elif ': ' in msg and msg.split(': ', 1)[0] in cleanup_actions.keys():
|
|
cleanup_action, msg_ = msg.split(': ', 1)
|
|
res['action'] = 'cleanup'
|
|
res['cleanup_action'] = cleanup_action
|
|
if cleanup_actions.get(cleanup_action):
|
|
res['action'] = cleanup_actions.get(cleanup_action)
|
|
msg_ = _strip_pattern(msg_, res, 'cleanup_context', pos='l')
|
|
parts = regexps['cleanup_optional_text'].split(msg_)
|
|
if len(parts) > 1: # {optional_text} is present
|
|
res['cleanup_optional_text'] = parts[1]
|
|
msg_ = msg_[: -len(parts[1]) - 2]
|
|
for field_name in ('helo', 'proto', 'to', 'from'):
|
|
msg_ = _strip_pattern(msg_, res, field_name, pos='r')
|
|
msg_ = _strip(msg_, ';', 'r')
|
|
if msg_.endswith('unknown'):
|
|
msg_ = msg_[:-8]
|
|
else:
|
|
msg_ = _strip_host_ip(msg_, res, pos='r')
|
|
msg_ = _strip(msg_, ' from', 'r')
|
|
res['text'] = msg_
|
|
|
|
# cleanup/cleanup_message.c:626,724-731 cleanup_header_callback
|
|
elif msg.startswith('message-id='):
|
|
res['message_id'] = msg[11:].lstrip('<').rstrip('>')
|
|
res['action'] = 'message_id'
|
|
|
|
# cleanup/cleanup_message.c:628,724-731 cleanup_header_callback
|
|
elif msg.startswith('resent-message-id='):
|
|
res['resent_message_id'] = msg[18:].lstrip('<').rstrip('>')
|
|
res['action'] = 'resent_message_id'
|
|
|
|
# cleanup/cleanup_milter.c:2066 cleanup_milter_apply
|
|
# {milter_action}: {where} from {hostname}[{ip}]: {text}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>
|
|
elif (
|
|
msg.startswith('milter-reject: ')
|
|
or msg.startswith('milter-discard: ')
|
|
or msg.startswith('milter-hold: ')
|
|
):
|
|
act_, msg_ = msg.split(': ', 1)
|
|
res['action'] = 'milter_action'
|
|
res['milter_action'] = act_[7:]
|
|
for event in cleanup_milter_apply_events:
|
|
if msg_.startswith(event):
|
|
res['milter_event'] = event
|
|
msg_ = msg_[len(event):]
|
|
break
|
|
for field_name in ('helo', 'proto', 'to', 'from'):
|
|
msg_ = _strip_pattern(msg_, res, field_name, pos='r')
|
|
msg_ = _strip(msg_, ';', 'r')
|
|
msg_ = _strip(msg_, ' from ', 'l')
|
|
msg_ = _strip_host_ip(msg_, res, pos='l')
|
|
msg_ = _strip(msg_, ': ', 'l')
|
|
res['text'] = msg_
|
|
|
|
# cleanup/cleanup_milter.c:252 cleanup_milter_hbc_log
|
|
# milter-{where}-{cleanup_action}: {where} {text} from {hostname}[{ip}]; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>: {optional_text}
|
|
elif msg.startswith('milter-'):
|
|
msg_ = msg[7:]
|
|
for wh in where:
|
|
if msg_.startswith(wh):
|
|
res['where'] = wh
|
|
break
|
|
if where in res:
|
|
msg_ = msg_[1:]
|
|
for cleanup_action in cleanup_actions.keys():
|
|
if msg_.startswith(cleanup_action):
|
|
res['cleanup_action'] = cleanup_action
|
|
if cleanup_actions.get(cleanup_action):
|
|
res['action'] = cleanup_actions.get(cleanup_action)
|
|
break
|
|
if 'cleanup_action' in res:
|
|
msg_ = _strip(msg_, ': ' + res['where'], 'l')
|
|
parts = regexps['cleanup_optional_text'].split(msg_)
|
|
if len(parts) > 1: # {optional_text} is present
|
|
res['cleanup_optional_text'] = parts[1]
|
|
msg_ = msg_[: -len(parts[1]) - 2]
|
|
for field_name in ('helo', 'proto', 'to', 'from'):
|
|
msg_ = _strip_pattern(msg_, res, field_name, pos='r')
|
|
msg_ = _strip(msg_, ';', 'r')
|
|
msg_ = _strip_host_ip(msg_, res, post='r')
|
|
msg_ = _strip(msg_, ' from', 'r')
|
|
res['text'] = msg_
|
|
if 'action' not in res:
|
|
res['action'] = 'milter_cleanup'
|
|
else:
|
|
res['parsed'] = False
|
|
else:
|
|
res['parsed'] = False
|
|
|
|
# cleanup/cleanup_milter.c:2538
|
|
# closing: {text}
|
|
# -> ignore (because no queue_id)
|
|
|
|
# cleanup/cleanup_milter.c:2557
|
|
# ignoring: {text}
|
|
# -> ignore (because no queue_id)
|
|
|
|
# cleanup/cleanup_milter.c:2684
|
|
# flags = {text}
|
|
# -> ignore (because no queue_id)
|
|
|
|
# cleanup/cleanup_milter.c:2686
|
|
# errs = {text}
|
|
# -> ignore (because no queue_id)
|
|
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_trivial_rewrite(msg, res):
|
|
"""
|
|
Parse log messages of the trivial-rewrite component.
|
|
|
|
Currently there is no relevant logging with level msg_info,
|
|
so we ignore all messages.
|
|
"""
|
|
res['action'] = 'ignore'
|
|
|
|
|
|
def _parse_smtp(msg, res):
|
|
"""
|
|
Parse log messages of the smtp component.
|
|
"""
|
|
# Logging information is often added to a DSB (delivery status buffer),
|
|
# more precisely to dsb->why and text to dsb->why->reason; adding is
|
|
# done with functions like dsb_simple, dsb_formal, dsb_update from
|
|
# global/dsn_buf.c; use `rg -A3 -B3 dsb_` to find this stuff.
|
|
#
|
|
# Other logging is done by calling log_adhoc, let's handle this and
|
|
# special cases first:
|
|
|
|
# global/log_adhoc.c:82-214 log_adhoc
|
|
# to=<{rcpt}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
# to=<{rcpt}>, orig_to=<{orig_to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
# 1) global/defer.c:267 status=deferred
|
|
# example text: lost connection with host.example.com[2606:2800:220:1:248:1893:25c8:1946] while receiving the initial server greeting
|
|
# (defer_append() is called in smtp/smtp_trouble.c:264-267,404-407)
|
|
# 2) global/bounce.c:322 status=SOFTBOUNCE
|
|
# example text: unknown user: "peter"
|
|
# 3) global/bounce.c:322,512 status=bounced
|
|
# example text: host host.example.com[2606:2800:220:1:248:1893:25c8:1946] said: 500 need some sleep (in reply to DATA command)
|
|
# (bounce_append() is called in smtp/smtp_trouble.c:264-267,404-407)
|
|
# 4) global/sent.c:162 status=sent
|
|
# example text: 250 2.0.0 Ok: queued as D9A33901180
|
|
# (sent() is called in smtp/smtp_rcpt.c:175-177)
|
|
if msg.startswith('to=<'):
|
|
_parse_log_adhoc(msg, res) # sets action to 'delivery_status'
|
|
|
|
# smtp/smtp_proto.c:417
|
|
# enabling PIX workarounds: %s for %s
|
|
elif msg.startswith('enabling PIX workarounds: '):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_connect.c:1057
|
|
|
|
# smtp/smtp_proto.c:1144-1175 smtp_hbc_logger (hbc = header body checks)
|
|
# smtp/smtp_proto.c:266-269 smtp_hbc_callbacks
|
|
# smtp/smtp.c:1326-1333 hbc_header_checks_create,hbc_body_checks_create
|
|
# global/header_body_checks.c:366-415 return hbc where hbc->call_backs contains the logging callback
|
|
# the hbc struct is used in :1245-1274(smtp_header_rewrite) and the
|
|
# smtp main protocol loop smtp_loop (line 2250 calling mime_state_alloc)
|
|
# by calling hbc_header_checks
|
|
# global/header_body_checks.c:230-305 hbc_action contains the actual call of the logging callback
|
|
# here we have these actions (corresponding to some of the ACTIONS in man 5 header_checks):
|
|
# 255: warning, 259: info, 272: replace, 284: prepend, 290: strip
|
|
# (see global variable smtp_hbc_actions)
|
|
# {smtp_hbc_action}: {header_or_body} {content}: {text}
|
|
# {smtp_hbc_action}: {header_or_body} {content}
|
|
# -> ignore
|
|
elif msg.split(': ', 1)[0] in smtp_hbc_actions:
|
|
res['action'] = 'ignore'
|
|
|
|
elif msg.startswith('host ') or msg.startswith('Protocol error: host '):
|
|
if msg.startswith('Protocol error: '):
|
|
msg_ = msg[21:]
|
|
res['smtp_protocol_error'] = True
|
|
else:
|
|
msg_ = msg[5:]
|
|
sep1 = ' refused to talk to me: '
|
|
sep2 = ' said: '
|
|
if sep1 in msg:
|
|
# smtp/smtp_proto.c:375-378,472-475,484-487,493-496
|
|
# host {hostname_or_unknown}[{ip_address}] refused to talk to me: {error_detail}
|
|
host_ip, smtp_error = msg_.split(sep1, 1)
|
|
res['smtp_error'] = smtp_error
|
|
_strip_host_ip(host_ip, res, pos='l')
|
|
res['action'] = 'smtp_error'
|
|
elif sep2 in msg:
|
|
# smtp/smtp_proto.c:1940-1944,2013-2017,2037-2041,2084-2088,2111-2115
|
|
# host {hostname_or_unknown}[{ip_address}] said: {error_text} (in reply to {command})
|
|
# (calls smtp_rcpt_fail or smtp_mesg_fail in smtp/smtp_trouble.c
|
|
# where vsmtp_fill_dsn fills state->why->reason with the
|
|
# formatted string, possibly prepending 'Protocol error: ')
|
|
host_ip, smtp_error = msg_.split(sep2, 1)
|
|
res['smtp_error'] = smtp_error
|
|
_strip_host_ip(host_ip, res, pos='l')
|
|
res['action'] = 'smtp_error'
|
|
else:
|
|
res['parsed'] = False
|
|
|
|
# smtp/smtp_connect.c:1057
|
|
# smtp/smtp_connect.c:997,195 (call to smtp_connect_addr)
|
|
# network address conversion failed: %m
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:1057
|
|
# smtp/smtp_connect.c:974 (call to smtp_tls_policy_cache_query) TODO
|
|
|
|
# smtp/smtp_proto.c:839
|
|
# smtp/smtp_sasl_proto.c:173-175
|
|
# SASL authentication failed: server %s offered no compatible authentication mechanisms for this type of connection security
|
|
# -> ignore
|
|
|
|
# smtp/smtp_sasl_glue.c:403-405
|
|
# SASL authentication failed; authentication protocol loop with server %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_addr.c:216-220
|
|
# unable to look up host %s: %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_addr.c:236-237
|
|
# {host}: host not found
|
|
# -> ignore
|
|
|
|
# smtp/smtp_addr.c:609-610
|
|
# unable to find primary relay for %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_addr.c:612-613,677
|
|
# mail for %s loops back to myself
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:141
|
|
# Server configuration error
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:195
|
|
# network address conversion failed: %m
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:308-309
|
|
# connect to %s[%s]:%d: %m
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:311
|
|
# connect to %s[%s]: %m
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:782
|
|
# all network protocols are disabled
|
|
# -> ignore
|
|
|
|
# smtp/smtp_connect.c:1085-1086
|
|
# server unavailable or unable to receive mail
|
|
# -> ignore
|
|
|
|
# smtp/smtp_tls_policy.c:168
|
|
# client TLS configuration problem
|
|
# -> ignore
|
|
|
|
# smtp/smtp_tls_policy.c:209
|
|
# Temporary lookup error
|
|
# -> ignore
|
|
|
|
# smtp/smtp_tls_policy.c:861-862
|
|
# TLSA lookup error for %s:%u
|
|
# -> ignore
|
|
|
|
# # dsb_simple, vdsb_simple
|
|
|
|
# calls to smtp/smtp_tls_policy.c:769 dane_incopmat (vdsb_simple in lin 789): TODO
|
|
|
|
# # dsb_update
|
|
|
|
# smtp/smtp_tls_policy.c:714-718 TODO
|
|
|
|
# smtp/smtp_sasl_glue.c:359-363
|
|
# SASL authentication failed; cannot authenticate to server %s: %s
|
|
elif msg.startswith(
|
|
'SASL authentication failed; cannot authenticate to server '
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_sasl_glue.c:422-426
|
|
# SASL authentication failed; non-empty initial %s challenge from server %s: %s
|
|
elif msg.startswith('SASL authentication failed; non-empty initial '):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_sasl_glue.c:433-437
|
|
# SASL authentication failed; cannot authenticate to server %s: %s
|
|
elif msg.startswith(
|
|
'SASL authentication failed; cannot authenticate to server '
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_sasl_glue.c:460-464
|
|
# SASL authentication failed; server %s said: %s
|
|
elif (
|
|
msg.startswith('SASL authentication failed; server ')
|
|
and ' said: ' in msg
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_sasl_glue.c:342-345
|
|
# SASL [CACHED] authentication failed; server %s said: %s
|
|
# and
|
|
# smtp/smtp_sasl_proto.c:192-193 smtp_sasl_helo_login calling smtp/smtp_sasl_glue.c:311 smtp_sasl_authenticate
|
|
# SASL [CACHED] authentication failed; server %s said: %s
|
|
elif msg.startswith('SASL [CACHED] authentication failed; server '):
|
|
res['action'] = 'ignore'
|
|
|
|
# # smtp_sess_fail
|
|
|
|
# smtp/smtp_sasl_proto.c:173-175 smtp_sasl_helo_login
|
|
# SASL authentication failed: server %s offered no compatible authentication mechanisms for this type of connection security
|
|
elif msg.startswith(
|
|
'SASL authentication failed: server '
|
|
) and msg.endswith(
|
|
' offered no compatible authentication mechanisms '
|
|
'for this type of connection security'
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_sasl_proto.c:192-193 smtp_sasl_helo_login calling smtp/smtp_sasl_glue.c:311 smtp_sasl_authenticate
|
|
# SASL [CACHED] authentication failed; server %s said: %s
|
|
# -> already done
|
|
|
|
# smtp/smtp_connect.c:1196 prevents smtp/smtp_trouble.c:224-225 and thus
|
|
# defer_append or bounce_append get called in lines 264-267, which both
|
|
# call log_adhoc and are covered above
|
|
# -> already done
|
|
|
|
# # smtp_bulk_fail
|
|
|
|
# smtp/smtp_trouble.c:435,461 (calling smtp_bulk_fail calling msg_info in line 226)
|
|
# lost connection with {hostname_or_unknown}[{ip_address}] while {text}
|
|
# example text: receiving the initial server greeting
|
|
# example text: performing the EHLO handshake
|
|
# example text: performing the HELO handshake
|
|
# example text: performing the LHLO handshake
|
|
# example text: sending MAIL FROM
|
|
# example text: sending RCPT TO
|
|
# example text: receiving the STARTTLS response
|
|
elif msg.startswith('lost connection with '):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_trouble.c:443,461 (calling smtp_bulk_fail calling msg_info in line 226)
|
|
# conversation with {hostname_or_unknown}[{ip_address}] timed out while {text}
|
|
elif msg.startswith('conversation with '):
|
|
res['action'] = 'ignore'
|
|
|
|
# smtp/smtp_trouble.c:452,461 (calling smtp_bulk_fail calling msg_info in line 226)
|
|
# local data error while talking to {hostname_or_unknown}[{ip_address}]
|
|
elif msg.startswith('local data error while talking to '):
|
|
res['action'] = 'ignore'
|
|
|
|
# # smtp_site_fail
|
|
|
|
# smtp/smtp_proto.c:388-390
|
|
# client TLS configuration problem
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:555-558,560-563
|
|
# mail for {nexthop} loops back to myself
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:803-806
|
|
# TLS is required, but host %s refused to start TLS: %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:819-822
|
|
# TLS is required, but was not offered by host %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:824-826
|
|
# TLS is required, but our TLS engine is unavailable
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:830-832
|
|
# TLS is required, but unavailable
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:1124-1126
|
|
# Server certificate not trusted
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:1129-1131
|
|
# Server certificate not verified
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:1960-1962
|
|
# unexpected server message
|
|
# -> ignore
|
|
|
|
# # smtp_mesg_fail
|
|
|
|
# smtp/smtp_proto.c:650-654
|
|
# SMTPUTF8 is required, but was not offered by host %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:1386-1388
|
|
# %s
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:2312-2314
|
|
# unreadable mail queue entry
|
|
# -> ignore
|
|
|
|
# smtp/smtp_proto.c:2388-2392
|
|
# message size %lu exceeds size limit %.0f of server %s
|
|
# -> ignore
|
|
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_bounce(msg, res):
|
|
"""
|
|
Parse log messages of the bounce component.
|
|
"""
|
|
# Logging is mainly done with msg_info.
|
|
|
|
# bounce/bounce_notify_service.c:202-203,290-291
|
|
# bounce/bounce_notify_verp.c:239-240
|
|
# bounce/bounce_one_service.c:168-169,244-245
|
|
# postmaster non-delivery notification: {queue_id}
|
|
if msg.startswith('postmaster non-delivery notification: '):
|
|
_strip_queue_id(msg, res, pos='r', target_name='bounce_id')
|
|
res['action'] = 'bounce_final'
|
|
|
|
# bounce/bounce_notify_service.c:243-244
|
|
# bounce/bounce_notify_verp.c:187-188
|
|
# bounce/bounce_one_service.c:206-207
|
|
# sender non-delivery notification: {queue_id}
|
|
elif msg.startswith('sender non-delivery notification: '):
|
|
_strip_queue_id(msg, res, pos='r', target_name='bounce_id')
|
|
res['action'] = 'bounce_final'
|
|
|
|
# bounce/bounce_warn_service.c:191-192,274-275
|
|
# postmaster delay notification: {queue_id}
|
|
elif msg.startswith('postmaster delay notification: '):
|
|
_strip_queue_id(msg, res, pos='r', target_name='bounce_id')
|
|
res['action'] = 'bounce_delay'
|
|
|
|
# bounce/bounce_warn_service.c:230-231
|
|
# sender delay notification: {queue_id}
|
|
elif msg.startswith('sender delay notification: '):
|
|
_strip_queue_id(msg, res, pos='r', target_name='bounce_id')
|
|
res['action'] = 'bounce_delay'
|
|
|
|
# bounce/bounce_trace_service.c:107-108
|
|
# not sending trace/success notification for double-bounce message
|
|
elif (
|
|
msg
|
|
== 'not sending trace/success notification for double-bounce message'
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# bounce/bounce_trace_service.c:115-116
|
|
# not sending trace/success notification for single-bounce message
|
|
elif (
|
|
msg
|
|
== 'not sending trace/success notification for single-bounce message'
|
|
):
|
|
res['action'] = 'ignore'
|
|
|
|
# bounce/bounce_trace_service.c:196-197
|
|
# sender delivery status notification: {queue_id}
|
|
elif msg.startswith('sender delivery status notification: '):
|
|
_strip_queue_id(msg, res, pos='r', target_name='bounce_id')
|
|
res['action'] = 'bounce_notify_sender'
|
|
|
|
# bounde/bounce_notify_verp.c:129
|
|
# -> ignore
|
|
|
|
# global/bounce.c:368,403
|
|
# status=deferred (bounce failed)
|
|
elif msg == 'status=deferred (bounce failed)':
|
|
res['action'] = 'bounce_failed'
|
|
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_virtual(msg, res):
|
|
"""
|
|
Parse log messages of the virtual component.
|
|
"""
|
|
# to=<{to}>, relay=virtual, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
if msg.startswith('to=<'):
|
|
_parse_log_adhoc(msg, res) # sets action to 'delivery_status'
|
|
|
|
# virtual/unknown.c:61-62
|
|
# unknown user: "{user}"
|
|
# -> ignore
|
|
|
|
# virtual/maildir.c:103
|
|
# delivers to maildir
|
|
# -> ignore
|
|
|
|
# virtual/maildir.c:186-187,211-212
|
|
# create maildir file %s: %m
|
|
# -> ignore
|
|
|
|
# virtual/maildir.c:242
|
|
# delivered to maildir
|
|
# -> ignore
|
|
|
|
# virtual/mailbox.c:101
|
|
# delivers to mailbox
|
|
# -> ignore
|
|
|
|
# virtual/mailbox.c:131,208,234,249,257
|
|
# mail system configuration error
|
|
# -> ignore
|
|
|
|
# virtual/mailbox.c:134-135
|
|
# destination %s is not owned by recipient
|
|
# -> ignore
|
|
|
|
# virtual/mailbox.c:163
|
|
# delivered to mailbox
|
|
# -> ignore
|
|
|
|
else:
|
|
res['parsed'] = False
|
|
|
|
|
|
def _parse_error(msg, res):
|
|
"""
|
|
Parse log messages of the error component.
|
|
"""
|
|
# to=<{to}>, relay=none, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text})
|
|
if msg.startswith('to=<'):
|
|
_parse_log_adhoc(msg, res) # sets action to 'delivery_status'
|
|
else:
|
|
res['action'] = 'ignore'
|
|
res['parsed'] = False
|
|
|
|
|
|
def extract_delivery(
|
|
msg_details: dict, parsed: dict
|
|
) -> Tuple[Optional[List[str]], Optional[dict]]:
|
|
"""
|
|
Compute a delivery item from a parsed entry.
|
|
|
|
Basically this means that we map the parsed fields to
|
|
a type ('from' or 'to') and to the database
|
|
fields for table 'delivery_from' or 'delivery_to'.
|
|
|
|
We branch to functions _extract_{comp} where comp is the
|
|
name of a Postfix component.
|
|
|
|
Return a list of error strings and a dict with the
|
|
extracted information. Keys with None values are removed
|
|
from the dict.
|
|
"""
|
|
comp = parsed['comp']
|
|
delivery = {
|
|
'type': 'to',
|
|
't': msg_details['t'],
|
|
'queue_id': parsed.get('queue_id'),
|
|
'message': msg_details['message'],
|
|
'identifier': msg_details['identifier'],
|
|
'pid': msg_details['pid'],
|
|
'comp': comp,
|
|
}
|
|
action = parsed.get('action')
|
|
if action == 'ignore' or action is None:
|
|
return None, None
|
|
elif action == 'subject':
|
|
delivery['type'] = 'from'
|
|
delivery['subject'] = parsed.get('subject')
|
|
elif comp == 'smtpd' or comp.endswith('/smtpd'):
|
|
delivery = _extract_smtpd(msg_details, parsed, delivery, action)
|
|
delivery['comp'] = 'smtpd'
|
|
elif comp == 'qmgr':
|
|
delivery = _extract_qmgr(msg_details, parsed, delivery, action)
|
|
elif comp == 'cleanup':
|
|
delivery = _extract_cleanup(msg_details, parsed, delivery, action)
|
|
elif comp == 'trivial-rewrite':
|
|
delivery = _extract_trivial_rewrite(
|
|
msg_details, parsed, delivery, action
|
|
)
|
|
elif comp in ('smtp', 'lmtp') or comp.endswith('/smtp'):
|
|
delivery = _extract_smtp(msg_details, parsed, delivery, action)
|
|
if delivery:
|
|
delivery['comp'] = 'smtp'
|
|
elif comp == 'bounce':
|
|
delivery = _extract_bounce(msg_details, parsed, delivery, action)
|
|
elif comp == 'virtual':
|
|
delivery = _extract_virtual(msg_details, parsed, delivery, action)
|
|
elif comp == 'error':
|
|
delivery = _extract_error(msg_details, parsed, delivery, action)
|
|
else:
|
|
return ['Cannot extract_delivery'], None
|
|
if 'verp_id' in parsed and delivery and delivery['type'] == 'from':
|
|
try:
|
|
delivery['verp_id'] = int(parsed['verp_id'])
|
|
except Exception:
|
|
pass
|
|
# remove keys with None values
|
|
if delivery:
|
|
delivery = {k: v for k, v in delivery.items() if v is not None}
|
|
return None, delivery
|
|
|
|
|
|
def _extract_smtpd(msg_details, parsed, delivery, action):
|
|
delivery['type'] = 'from'
|
|
if action == 'connect':
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['ip'] = parsed.get('ip')
|
|
delivery['sasl_username'] = parsed.get('sasl_username')
|
|
delivery['orig_queue_id'] = parsed.get('orig_queue_id')
|
|
elif action in ('reject', 'hold', 'delay', 'discard', 'redirect'):
|
|
# Note: Here we may have both sender and recipient.
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['ip'] = parsed.get('ip')
|
|
delivery['status'] = action
|
|
delivery['sasl_username'] = parsed.get('sasl_username')
|
|
delivery['sender'] = parsed.get('from')
|
|
delivery['recipient'] = parsed.get('to')
|
|
delivery['phase'] = parsed.get('where')
|
|
delivery['error'] = parsed.get('error')
|
|
else:
|
|
return None
|
|
return delivery
|
|
|
|
|
|
def _extract_qmgr(msg_details, parsed, delivery, action):
|
|
delivery['type'] = 'from'
|
|
delivery['status'] = action
|
|
if action == 'expired':
|
|
delivery['done'] = True
|
|
elif action == 'removed':
|
|
delivery['done'] = True
|
|
elif action == 'queued':
|
|
delivery['accepted'] = True
|
|
delivery['sender'] = parsed.get('from')
|
|
delivery['size'] = parsed.get('size')
|
|
delivery['nrcpt'] = parsed.get('nrcpt')
|
|
elif action == 'skipped':
|
|
delivery['accepted'] = False
|
|
delivery['done'] = True
|
|
else:
|
|
return None
|
|
return delivery
|
|
|
|
|
|
def _extract_cleanup(msg_details, parsed, delivery, action):
|
|
delivery['type'] = 'from'
|
|
if action == 'reject':
|
|
delivery['status'] = action
|
|
delivery['accepted'] = False
|
|
if 'mime_error' in parsed:
|
|
delivery['phase'] = 'mime'
|
|
delivery['error'] = (
|
|
parsed.get('mime_error', '?')
|
|
+ ': '
|
|
+ parsed.get('mime_error_detail', '?')
|
|
)
|
|
elif action in ('cleanup', 'hold', 'delay', 'discard', 'redirect'):
|
|
delivery['status'] = action
|
|
delivery['from'] = parsed.get('from')
|
|
delivery['to'] = parsed.get('to')
|
|
elif action == 'message_id':
|
|
delivery['message_id'] = parsed.get('message_id')
|
|
elif action == 'resent_message_id':
|
|
delivery['resent_message_id'] = parsed.get('resent_message_id')
|
|
elif action == 'milter_action':
|
|
return None
|
|
elif action == 'milter_cleanup':
|
|
return None
|
|
else:
|
|
return None
|
|
return delivery
|
|
|
|
|
|
def _extract_trivial_rewrite(msg_details, parsed, delivery, action):
|
|
return None
|
|
|
|
|
|
def _extract_smtp(msg_details, parsed, delivery, action):
|
|
if action == 'delivery_status':
|
|
delivery['recipient'] = parsed.get('to')
|
|
delivery['orig_recipient'] = parsed.get('orig_to')
|
|
delivery['relay'] = parsed.get('relay')
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['destination'] = parsed.get('destination')
|
|
delivery['port'] = parsed.get('port')
|
|
delivery['delay'] = parsed.get('delay')
|
|
delivery['delays'] = parsed.get('delays')
|
|
delivery['dsn'] = parsed.get('dsn')
|
|
delivery['status'] = parsed.get('delivery_status')
|
|
delivery['status_text'] = parsed.get('delivery_status_text')
|
|
elif action == 'smtp_error':
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['destination'] = parsed.get('ip')
|
|
delivery['status'] = 'smtp_error'
|
|
delivery['status_text'] = parsed.get('smtp_error')
|
|
else:
|
|
return None
|
|
return delivery
|
|
|
|
|
|
def _extract_bounce(msg_details, parsed, delivery, action):
|
|
if action == 'bounce_delay':
|
|
delivery['status'] = 'bounce_delay'
|
|
delivery['status_text'] = parsed.get('bounce_id')
|
|
elif action == 'bounce_final':
|
|
delivery['status'] = 'bounce_final'
|
|
delivery['status_text'] = parsed.get('bounce_id')
|
|
elif action == 'bounce_notify_sender':
|
|
delivery['status'] = 'bounce_notify_sender'
|
|
delivery['status_text'] = parsed.get('bounce_id')
|
|
elif action == 'bounce_failed':
|
|
delivery['status'] = 'bounce_failed'
|
|
else:
|
|
return None
|
|
|
|
|
|
def _extract_virtual(msg_details, parsed, delivery, action):
|
|
if action == 'delivery_status':
|
|
delivery['recipient'] = parsed.get('to')
|
|
delivery['orig_recipient'] = parsed.get('orig_to')
|
|
delivery['relay'] = parsed.get('relay')
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['destination'] = parsed.get('destination')
|
|
delivery['port'] = parsed.get('port')
|
|
delivery['delay'] = parsed.get('delay')
|
|
delivery['delays'] = parsed.get('delays')
|
|
delivery['dsn'] = parsed.get('dsn')
|
|
delivery['status'] = parsed.get('delivery_status')
|
|
delivery['status_text'] = parsed.get('delivery_status_text')
|
|
else:
|
|
return None
|
|
|
|
|
|
def _extract_error(msg_details, parsed, delivery, action):
|
|
if action == 'delivery_status':
|
|
delivery['recipient'] = parsed.get('to')
|
|
delivery['orig_recipient'] = parsed.get('orig_to')
|
|
delivery['relay'] = parsed.get('relay')
|
|
delivery['host'] = parsed.get('host')
|
|
delivery['destination'] = parsed.get('destination')
|
|
delivery['port'] = parsed.get('port')
|
|
delivery['delay'] = parsed.get('delay')
|
|
delivery['delays'] = parsed.get('delays')
|
|
delivery['dsn'] = parsed.get('dsn')
|
|
delivery['status'] = parsed.get('delivery_status')
|
|
delivery['status_text'] = parsed.get('delivery_status_text')
|
|
else:
|
|
return None
|