r""" Parse postfix log messages. The parser (:func:`parse_entry`) uses regular expressions to produce a dict. This dict is used by :func:`extract_delivery` to extract relevant mail delivery information. If VERP id parsing is to be used, call :func:`init` before using :func:`parse`. Comments like 'smtpd/smtpd.c:1663-1664' refer to a postfix-3.4.7 source file and line numbers within that file. We assume that verbose logging is off, which is the default (util/msg.c:177). We do not parse verbose messages (variable "msg_verbose", usually). Useful command for searching postfix sources (in subdirs global, smtpd, qmgr, ...): rg 'msg_info\("%s:' -B2 -n *.c We only parse messages of level info and some of level warning (see util/msg.c:67-71, also cf. postlog/postlog.c:140-150); messages of levels 'error', 'fatal', 'panic' are discarded. We only parse messages with queue_id (considering "NOQUEUE" as queue_id == None). Coverage of postfix daemon components: * smtpd: yes (includes submission) * trivial-rewrite: yes (no relevant messages of level msg_info) * cleanup: yes * qmgr: yes * smtp, lmtp: mostly * bounce: mostly * virtual: mostly * error: partially These components are not covered: * anvil * discard * dnsblog * flush * local * oqmgr * pickup * pipe * postlogd * postscreen * proxymap * qmqpd * scache * showq * tlsmgr * tlsproxy * verify Note: In particular, local delivery is not supported! For lmtp we try to extract a dovecot_id from the delivery status text, see :func:`_get_delivery_status`. """ import re from pprint import pprint from traceback import format_exc from typing import List, Optional, Tuple, Union ignore_identifiers = ( 'postfix/master', 'postfix/postfix-script', 'configure-instance.sh', 'postmulti', ) """ Syslog identifiers to ignore. """ where = ( 'CONNECT', 'DATA content', 'BDAT content', 'END-OF-MESSAGE', 'HELO', 'EHLO', 'STARTTLS', 'AUTH', 'MAIL', 'RCPT', 'DATA', 'BDAT', 'RSET', 'NOOP', 'VRFY', 'ETRN', 'QUIT', 'XCLIENT', 'XFORWARD', 'UNKNOWN', ) """ Possible smtpd "where" values from smtpd/smtpd.c:235-260. """ smtpd_whatsup_actions = { 'reject': 'reject', 'hangup': 'reject', 'info': None, 'warn': None, 'filter': None, 'hold': 'hold', 'delay': 'delay', 'discard': 'discard', 'redirect': 'redirect', 'bcc': None, 'permit': None, 'reject_warning': 'reject', } """ Keys are from `rg log_whatsup smtpd/*.c` and smtpd/s,tpd_check.c:998,1038. Map the possible smtpd whatsup value to our action. """ cleanup_actions = { 'reject': 'reject', 'warning': None, 'info': None, 'filter': None, 'pass': None, 'discard': 'discard', 'hold': 'hold', 'delay': 'delay', 'prepend': None, 'replace': None, 'redirect': 'redirect', 'bcc': None, 'strip': None, } """ Possible cleanup "actions" and a mapping to our action. """ smtp_hbc_actions = ['warning', 'info', 'replace', 'prepend', 'strip'] """ SMTP header body check actions. """ cleanup_contexts = ('header', 'body', 'content') mime_error_texts = [ 'MIME nesting exceeds safety limit', 'message header length exceeds safety limit', 'improper use of 8-bit data in message header', 'improper use of 8-bit data in message body', 'invalid message/* or multipart/* encoding domain', ] """ MIME error texts in cleanup. """ # rfc3464_actions = ['failed', 'delayed', 'delivered', 'relayed', 'expanded'] address_pattern = r'([^">]*|"([^ "\\]|\\[^ ])*"@[^>]*)' """ Email address pattern. Either match any number of characters not containing '"' or '>', or match a local part followed by '@' and a domain part, where the domain part is arbitrary, but must not contain '>', and the local part begins and ends with a '"' and contains 1) any char except space, tab, '"', r'\' or 2) any char except tab prepended by r'\'. Note: * email addresses are by default logged in 'external' format by Postfix >=3.5: http://www.postfix.org/postconf.5.html#info_log_address_format * https://stackoverflow.com/questions/201323/how-to-validate-an-email-address-using-a-regular-expression """ regexp_patterns = { 'queue_id_short': r'([0-9A-F]{12})', 'queue_id_long': r'([0-9BCDFGHIJKLMNPQRSTVWXYZbcdfghijklmnpqrstvwxyz]{10,15}z[0-9BCDFGHIJKLMNPQRSTVWXYZbcdfghijklmnpqrstvwxy]{5,10})', 'from': 'from=<' + address_pattern + '>', 'to': 'to=<' + address_pattern + '>', 'orig_to': 'orig_to=<' + address_pattern + '>', 'message-id': r'message-id=<([^>]*)>', 'nrcpt': r'nrcpt=([0-9]+)', 'relay': r'relay=(none|local|virtual|([^ ]+)\[([^\]]+)\](:([0-9]+))?)', # matches 5 values 'delay': r'delay=([0-9\.]+)', 'delays': r'delays=([0-9\./]+)', 'dsn': r'dsn=([0-9\.]+)', 'proto': r'proto=(SMTP|ESMTP)', 'helo': r'helo=<([^>]*)>', 'host_ip': r'([^ ]+)\[([^\]]+)\]', # matches 2 values 'none': r'none', 'dovecot_id': r'([A-za-z0-9/\+]{22}) Saved', 'sasl_username': r'sasl_username=?', 'sasl_method': r'sasl_method=(.*)', 'size': r'size=([0-9]+)', 'orig_client': r'orig_client=(.*)', 'orig_queue_id': r'orig_queue_id=(.*)', 'sasl_sender': r'sasl_sender=(.*)', 'cleanup_context': '(' + '|'.join(cleanup_contexts) + ')', } """ regexp patterns, usually for matching one expression. """ cleanup_milter_apply_events = ( 'END-OF-MESSAGE', 'CONNECT', 'EHLO', 'MAIL', 'RCPT', 'DATA', ) """ cleanup milter stages. """ regexps = { 'cleanup_optional_text': re.compile(' helo=<[^>]*>: '), 'failed_mail': re.compile( r'^([^\[]+)\[([^\]]+)\]: (.*); proto=[^ ]+ helo=<([^>]+)>$' ), 'failed_rcpt': re.compile( r'^([^\[]+)\[([^\]]+)\]: (.*); from=<([^>]*)> to=<([^>]+)> proto=[^ ]+ helo=<([^>]+)>$' ), } """ Special regular expressions for matching expressions which are harder to parse. """ for label, pattern in regexp_patterns.items(): regexps['l_' + label] = re.compile(r'(^' + pattern + r'(: |, | |$))') regexps['r_' + label] = re.compile(r'( ?' + pattern + r'$)') def _strip_queue_id(msg, res, pos='l', target_name='queue_id'): """ Strip a postfix queue_id at the left or right end of *msg*. The queue_id can either be a short one, or a long one. If none is matched, return *msg*. """ m = regexps[pos + '_queue_id_short'].search(msg) if not m: m = regexps[pos + '_queue_id_long'].search(msg) if not m: return msg res[target_name] = m.group(2) l_ = len(m.group(1)) return msg[l_:] if pos == 'l' else msg[:-l_] def _strip_pattern(msg, res, pattern_name, pos='l', target_names=None): """ Strip a pattern at the left/right end of *msg* and store fields. Matching at the left (right) end is chosen be pos='l' (pos='r'). *pattern_name* is a key from regexp_patterns. If target_names is set, it must be an iterable. Each name in it is used as a key in *res* and the values are set to values matched by the pattern. Most patterns in regexp_patterns only match one value; in this case the by default (target_names=None) the *pattern_name* will be used as key in *res*. """ m = regexps[pos + '_' + pattern_name].search(msg) if m: if target_names is None: target_names = (pattern_name,) for ind, target_name in enumerate(target_names): res[target_name] = m.group(2 + ind) l_ = len(m.group(1)) msg = msg[l_:] if pos == 'l' else msg[:-l_] return msg def _rstrip_smtpd_whatsup(msg, res): """ Strip and store smtpd_whatsup fields at the right end of *msg*. Return the msg part in front of them. """ # from=<{sender}> to=<{recipient}> proto=ESMTP helo=<{helo}> sasl_username=<{sasl_username}> for field_name in ('sasl_username', 'helo', 'proto', 'to', 'from'): msg = _strip_pattern(msg, res, field_name, pos='r') return msg def _strip_host_ip(msg, res, pos='l', allow_none=False): """ Strip a hostname followed by an IP address in brackets from *msg*. *pos* determines whether the pattern is matched on the left or right hand side of *msg*. The hostname is put into res['host'], the IP address into res['ip']. If the hostname equals "unknown", it is set to None. If allow_none evaluates to True and msg == 'none', this results in res['host'] = None and res['ip'] = None. """ if allow_none: msg_ = _strip_pattern(msg, res, 'none', pos=pos) if msg_ != msg: # "none" was matched res['host'] = None res['ip'] = None return msg_ msg = _strip_pattern( msg, res, 'host_ip', pos=pos, target_names=('host', 'ip') ) if res.get('host') == 'unknown': res['host'] = None return msg def _strip_relay(msg, res, pos='l'): """ Strip a relay pattern from *msg*. If "relay=none" or "relay=local" or "relay=virtual" we set host,destination,port in res to None. The destination can be an IP address or text (e.g., "private/dovecot-lmtp"). """ msg = _strip_pattern( msg, res, 'relay', pos=pos, target_names=('relay_full', 'host', 'destination', 'port_', 'port'), ) if res.get('relay_full') in ('none', 'local', 'virtual'): res['relay'] = res.get('relay_full') res['host'] = None res['destination'] = None res['port'] = None else: res['relay'] = 'external' if res['destination'] == 'private/dovecot-lmtp': res['relay'] = 'lmtp' return msg def _lstrip_where(msg, res): """ Strip and store 'where' string at the left end of *msg*. Return *msg*, where the 'where' string was strippped, if found. See global variable 'where'. """ for where_ in where: if msg.startswith(where_): res['where'] = where_ return msg[len(where_):] return msg def _strip(text, part, pos): """ Strip *part* from *text* on the left (pos='l') or right (pos='r') hand side. """ if pos == 'l': if text.startswith(part): return text[len(part):] if pos == 'r': if text.endswith(part): return text[:-len(part)] return text def _parse_log_adhoc(msg, res): """ Parse a log message formatted in global/log_adhoc.c:82-214 (log_adhoc). The message has one of these two formats: to=<{to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) to=<{to}>, orig_to=<{orig_to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) The resulting action will be 'delivery_status'. """ if msg.startswith('to=<'): msg_ = _strip_pattern(msg, res, 'to', pos='l') msg_ = _strip_pattern(msg_, res, 'orig_to', pos='l') msg_ = _strip_relay(msg_, res, pos='l') msg_ = _strip_pattern(msg_, res, 'delay', pos='l') msg_ = _strip_pattern(msg_, res, 'delays', pos='l') msg_ = _strip_pattern(msg_, res, 'dsn', pos='l') _get_delivery_status(msg_, res) if 'delivery_status' not in res: res['parsed'] = False return msg def _get_delivery_status(msg, res): """ Extract the delivery status from the beginning of *msg* into *res*. A delivery status looks like this: status={status} ({text}) Here {status} is a word (has no spaces). The result is put into res['delivery_status'] and res['delivery_status_text']. """ if msg.startswith('status='): res['action'] = 'delivery_status' if ' ' in msg: status, detail = msg.split(' ', 1) res['delivery_status'] = status[7:] res['delivery_status_text'] = detail.lstrip('(').rstrip(')') # also try to extract a dovecot_id; example: # 250 2.0.0 9y4WEP+qV11uBgAAZU03Dg Saved _strip_pattern( res['delivery_status_text'], res, 'dovecot_id', pos='r' ) else: # can this happen at all? res['delivery_status'] = msg[7:] return msg def init_parser(verp_marker: Optional[str] = None): """ Init the module. *verp_marker* is the VERP marker. """ global regexps if verp_marker: regexps['verp_id'] = re.compile( r'(.*\+.*)' + verp_marker + r'-(\d+)(@[^@]+$)' ) def parse_entry( msg_details: dict, debug: Union[bool, List[str]] = False ) -> Optional[dict]: """ Parse a log message returning a dict. *msg_details* is assumed to be a dict with these keys: * 'identifier' (syslog identifier), * 'pid' (process id), * 'message' (message text) The syslog identifier and process id are copied to the resulting dict. """ identifier = msg_details['identifier'] if identifier in ignore_identifiers: return None pid = msg_details['pid'] message = msg_details['message'].strip() # postfix component component = ( identifier[8:] if identifier.startswith('postfix/') else identifier ) res = {'comp': component, 'pid': pid} # do we have a postfix queue identifer? if message.startswith('NOQUEUE: '): res['queue_id'] = None msg_ = message[9:] else: # do not put key 'queue_id' into res, if not found msg_ = _strip_queue_id(message, res) try: if 'queue_id' in res: _parse_branch(component, msg_, res) except Exception: res['parsed'] = False print('PARSING FAILED:', message) print(format_exc()) res['parsed'] = not res.get('parsed') is False if debug: if not res['parsed']: print('-' * 20, message) print(component) elif 'queue_id' in res and res.get('action') == 'ignore': print('I' * 20, message) print(component) else: print(message) pprint(res) print('_' * 100) return res def _parse_branch(comp: str, msg: str, res: dict) -> None: """ Parse a log message string *msg*, adding results to dict *res*. Depending on the component *comp* we branch to functions named _parse_{comp}. Add parsing results to dict *res*. Always add key 'action'. Try to parse every syntactical element. Note: We parse what we can. Assessment of parsing results relevant for delivery is done in :func:`extract_delivery`. """ if ( msg.startswith('warning: ') or msg.startswith('error: ') or msg.startswith('fatal: ') or msg.startswith('panic: ') or msg.startswith('using backwards-compatible default setting') ): res['action'] = 'ignore' return if comp == 'smtpd' or comp.endswith( '/smtpd' ): # includes 'submission/smtpd' _parse_smtpd(msg, res) res['submission'] = comp.startswith('submission') elif comp == 'qmgr': _parse_qmgr(msg, res) elif comp == 'cleanup': _parse_cleanup(msg, res) elif comp == 'trivial-rewrite': _parse_trivial_rewrite(msg, res) elif comp in ('smtp', 'lmtp') or comp.endswith( '/smtp' ): # includes 'relay/smtp' _parse_smtp(msg, res) res['smtp_relay'] = comp.startswith('relay') elif comp == 'bounce': _parse_bounce(msg, res) elif comp == 'virtual': _parse_virtual(msg, res) elif comp == 'error': _parse_error(msg, res) else: res['parsed'] = False # extract a possible verp_id from orig_to if 'orig_to' in res: res['verp_id'], res['orig_to'] = _parse_verp_id(res['orig_to']) elif 'from' in res: res['verp_id'], res['from'] = _parse_verp_id(res['from']) def _parse_verp_id(email): """ Return th VERP id and the original email. """ if 'verp_id' in regexps: m = regexps['verp_id'].match(email) if m: verp_id = m.group(2) orig_email = m.group(1).rstrip('+') + m.group(3) return verp_id, orig_email return None, email def _parse_smtpd(msg, res): """ Parse log messages of the smtpd component, including submission. """ # smtpd/smtpd.c:2229-2246 smtpd_sasl_auth_cmd_wrapper # client={hostname_or_unknown}[{ip_address}] # client={hostname_or_unknown}[{ip_address}], sasl_method={sasl_method}, sasl_username={sasl_username}, sasl_sender={sasl_sender}, orig_queue_id={orig_queue_id}, orig_client={orig_client} # (sasl_* and orig_* are individually optional) if msg.startswith('client='): msg_ = _strip_host_ip(msg[7:], res, pos='l') msg_ = _strip_pattern(msg_, res, 'orig_client', pos='r') msg_ = _strip_pattern(msg_, res, 'orig_queue_id', pos='r') msg_ = _strip_pattern(msg_, res, 'sasl_sender', pos='r') msg_ = _strip_pattern(msg_, res, 'sasl_username', pos='r') msg_ = _strip_pattern(msg_, res, 'sasl_method', pos='r') res['action'] = 'connect' # smtpd/smtpd_check.c:949-967 log_whatsup # {smtpd_whatsup}: {where} from {hostname_or_unknown}[{ip_address}]: {error}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}> # smtpd/smtpd.c:5411-5414 smtpd_proto # reject: {where} from {hostname_or_unknown}[{ip_address}]: 421 4.3.0 {myhostname} Server local data error # reject: {where} from {hostname_or_unknown}[{ip_address}]: {error} elif ': ' in msg and msg.split(': ', 1)[0] in smtpd_whatsup_actions.keys(): smtpd_whatsup, msg_ = msg.split(': ', 1) res['smtpd_whatsup'] = smtpd_whatsup if smtpd_whatsup_actions.get(smtpd_whatsup): res['action'] = smtpd_whatsup_actions.get(smtpd_whatsup) else: res['action'] = 'ignore' msg_ = _lstrip_where(msg_, res) msg_ = _strip(msg_, ' from ', 'l') msg_ = _strip_host_ip(msg_, res, pos='l') msg_ = _strip(msg_, ': ', 'l') msg_ = _rstrip_smtpd_whatsup(msg_, res) msg_ = _strip(msg_, ';', 'r') res['error'] = msg_ # smtpd/smtpd.c:1663-1664 check_milter_reply # {milter_action}: {where} from {hostname_or_unknown}[{ip_address}]: {error}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}> elif msg.startswith('milter-hold: '): msg_ = _strip(_rstrip_smtpd_whatsup(msg[13:], res), ';', 'r') msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l') msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l') res['error'] = msg_ res['action'] = 'hold' elif msg.startswith('milter-discard: '): msg_ = _strip(_rstrip_smtpd_whatsup(msg[16:], res), ';', 'r') msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l') msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l') res['error'] = msg_ res['action'] = 'discard' elif msg.startswith('milter-reject: '): msg_ = _strip(_rstrip_smtpd_whatsup(msg[15:], res), ';', 'r') msg_ = _strip(_lstrip_where(msg_, res), ' from ', 'l') msg_ = _strip(_strip_host_ip(msg_, res, pos='l'), ': ', 'l') res['error'] = msg_ res['action'] = 'reject' # smtpd/smtpd.c:5099-5102 smtpd_start_tls # abort: TLS from {hostname_or_unknown}[{ip_address}]: {error} elif ( 'queue_id' in res and res['queue_id'] is None and msg.startswith('abort: TLS from ') ): msg_ = msg[16:] msg_ = _strip_host_ip(msg_, res, pos='l') msg_ = _strip(msg_, ': ', 'l') res['error'] = msg_ res['action'] = 'reject' # smtpd/smtpd.c:1850 ehlo_cmd # discarding EHLO keywords: {keywords} elif msg.startswith('discarding EHLO keywords: '): res['action'] = 'ignore' # smtpd/smtpd.c:5624 smtpd_proto # replacing command {command1} with {command2} # without queue_id # -> ignore else: res['action'] = 'ignore' res['parsed'] = False def _parse_qmgr(msg, res): """ Parse log messages of the qmgr component. """ # qmgr/qmgr_active.c:441 qmgr_active_done_25_generic # from=<{from}>, status=expired, returned to sender if msg.startswith('from=<') and msg.endswith( ', status=expired, returned to sender' ): res['from'] = msg[6:-36].rstrip('>') res['action'] = 'expired' # qmgr/qmgr_active.c:520 qmgr_active_done_3_generic elif msg == 'removed': res['action'] = 'removed' # qmgr/qmgr.c:680 pre_accept # %s: %s feedback type %d value at %d: %g # without queue_id # -> ignore # qmgr/qmgr_feedback.c:170 # "%s: %s feedback type %d value at %d: %g" # without queue_id # -> ignore # qmgr/qmgr_message.c:642-644 qmgr_message_read # global/opened.c:64,70,76,84-86 opened,vopened # from=<{from}>, size={size}, nrcpt={nrcpt} (queue {queue_name}) elif msg.startswith('from=<') and ', nrcpt=' in msg: msg_ = _strip_pattern(msg, res, 'from', pos='l') msg_ = _strip_pattern(msg_, res, 'size', pos='l') msg_ = _strip_pattern(msg_, res, 'nrcpt', pos='l') msg_ = _strip(msg_, '(queue ', 'l').rstrip(')') if msg_: res['queue_name'] = msg_ res['action'] = 'queued' # qmgr/qmgr_message.c:1493 qmgr_message_alloc - create in-core message structure elif msg == 'skipped, still being delivered': res['action'] = 'skipped' # qmgr/qmgr_queue.c:124-132 # "%s: feedback %g" # "%s: queue %s: limit %d window %d success %g failure %g fail_cohorts %g" # without queue_id # -> ignore else: res['action'] = 'ignore' res['parsed'] = False def _parse_cleanup(msg, res): """ Parse log messages of the cleanup component. """ if msg.startswith('info: '): msg_ = msg[6:] if msg_.lower().startswith('header subject: '): msg_ = _rstrip_smtpd_whatsup(msg_[16:], res).rstrip(';') res['subject'] = _strip( _strip_host_ip(msg_, res, pos='r'), ' from', 'r' ) res['action'] = 'subject' else: res['action'] = 'ignore' # cleanup/cleanup.c:584 elif msg.startswith('table ') and msg.endswith( ' has changed -- restarting' ): res['action'] = 'ignore' # cleanup/cleanup_masquerade.c:111 # "%s: %s map lookup problem -- " # -> ignore # cleanup/cleanup_message.c:1025 cleanup_mime_error_callback # reject: mime-error {mime_error}: {mime_error_detail} from {unknown_or_host_ip}; from=<{from}> to=<{to}> elif msg.startswith('reject: mime-error '): res['action'] = 'reject' msg_ = msg[19:] for mime_error_text in mime_error_texts: if msg_.startswith(mime_error_text): res['mime_error'] = mime_error_text msg_ = msg_[len(mime_error_text):] for field_name in ('to', 'from'): msg_ = _strip_pattern(msg_, res, field_name, pos='r') if msg_.endswith(' unknown'): msg_ = msg_[:-8] else: msg_ = _strip_host_ip(msg_, res, pos='r') msg_ = _strip(msg_, ' from', 'r') msg_ = _strip(msg_, ': ', 'l') if 'mime_error' in res: res['mime_error_detail'] = msg_ # cleanup/cleanup_message.c:255-276 cleanup_act_log # {cleanup_action}: {class} {content} from {attr}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>: {optional_text} elif ': ' in msg and msg.split(': ', 1)[0] in cleanup_actions.keys(): cleanup_action, msg_ = msg.split(': ', 1) res['action'] = 'cleanup' res['cleanup_action'] = cleanup_action if cleanup_actions.get(cleanup_action): res['action'] = cleanup_actions.get(cleanup_action) msg_ = _strip_pattern(msg_, res, 'cleanup_context', pos='l') parts = regexps['cleanup_optional_text'].split(msg_) if len(parts) > 1: # {optional_text} is present res['cleanup_optional_text'] = parts[1] msg_ = msg_[: -len(parts[1]) - 2] for field_name in ('helo', 'proto', 'to', 'from'): msg_ = _strip_pattern(msg_, res, field_name, pos='r') msg_ = _strip(msg_, ';', 'r') if msg_.endswith('unknown'): msg_ = msg_[:-8] else: msg_ = _strip_host_ip(msg_, res, pos='r') msg_ = _strip(msg_, ' from', 'r') res['text'] = msg_ # cleanup/cleanup_message.c:626,724-731 cleanup_header_callback elif msg.startswith('message-id='): res['message_id'] = msg[11:].lstrip('<').rstrip('>') res['action'] = 'message_id' # cleanup/cleanup_message.c:628,724-731 cleanup_header_callback elif msg.startswith('resent-message-id='): res['resent_message_id'] = msg[18:].lstrip('<').rstrip('>') res['action'] = 'resent_message_id' # cleanup/cleanup_milter.c:2066 cleanup_milter_apply # {milter_action}: {where} from {hostname}[{ip}]: {text}; from=<{from}> to=<{to}> proto={proto} helo=<{helo}> elif ( msg.startswith('milter-reject: ') or msg.startswith('milter-discard: ') or msg.startswith('milter-hold: ') ): act_, msg_ = msg.split(': ', 1) res['action'] = 'milter_action' res['milter_action'] = act_[7:] for event in cleanup_milter_apply_events: if msg_.startswith(event): res['milter_event'] = event msg_ = msg_[len(event):] break for field_name in ('helo', 'proto', 'to', 'from'): msg_ = _strip_pattern(msg_, res, field_name, pos='r') msg_ = _strip(msg_, ';', 'r') msg_ = _strip(msg_, ' from ', 'l') msg_ = _strip_host_ip(msg_, res, pos='l') msg_ = _strip(msg_, ': ', 'l') res['text'] = msg_ # cleanup/cleanup_milter.c:252 cleanup_milter_hbc_log # milter-{where}-{cleanup_action}: {where} {text} from {hostname}[{ip}]; from=<{from}> to=<{to}> proto={proto} helo=<{helo}>: {optional_text} elif msg.startswith('milter-'): msg_ = msg[7:] for wh in where: if msg_.startswith(wh): res['where'] = wh break if where in res: msg_ = msg_[1:] for cleanup_action in cleanup_actions.keys(): if msg_.startswith(cleanup_action): res['cleanup_action'] = cleanup_action if cleanup_actions.get(cleanup_action): res['action'] = cleanup_actions.get(cleanup_action) break if 'cleanup_action' in res: msg_ = _strip(msg_, ': ' + res['where'], 'l') parts = regexps['cleanup_optional_text'].split(msg_) if len(parts) > 1: # {optional_text} is present res['cleanup_optional_text'] = parts[1] msg_ = msg_[: -len(parts[1]) - 2] for field_name in ('helo', 'proto', 'to', 'from'): msg_ = _strip_pattern(msg_, res, field_name, pos='r') msg_ = _strip(msg_, ';', 'r') msg_ = _strip_host_ip(msg_, res, post='r') msg_ = _strip(msg_, ' from', 'r') res['text'] = msg_ if 'action' not in res: res['action'] = 'milter_cleanup' else: res['parsed'] = False else: res['parsed'] = False # cleanup/cleanup_milter.c:2538 # closing: {text} # -> ignore (because no queue_id) # cleanup/cleanup_milter.c:2557 # ignoring: {text} # -> ignore (because no queue_id) # cleanup/cleanup_milter.c:2684 # flags = {text} # -> ignore (because no queue_id) # cleanup/cleanup_milter.c:2686 # errs = {text} # -> ignore (because no queue_id) else: res['action'] = 'ignore' res['parsed'] = False def _parse_trivial_rewrite(msg, res): """ Parse log messages of the trivial-rewrite component. Currently there is no relevant logging with level msg_info, so we ignore all messages. """ res['action'] = 'ignore' def _parse_smtp(msg, res): """ Parse log messages of the smtp component. """ # Logging information is often added to a DSB (delivery status buffer), # more precisely to dsb->why and text to dsb->why->reason; adding is # done with functions like dsb_simple, dsb_formal, dsb_update from # global/dsn_buf.c; use `rg -A3 -B3 dsb_` to find this stuff. # # Other logging is done by calling log_adhoc, let's handle this and # special cases first: # global/log_adhoc.c:82-214 log_adhoc # to=<{rcpt}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) # to=<{rcpt}>, orig_to=<{orig_to}>, relay={none_or_host_ip_port}, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) # 1) global/defer.c:267 status=deferred # example text: lost connection with host.example.com[2606:2800:220:1:248:1893:25c8:1946] while receiving the initial server greeting # (defer_append() is called in smtp/smtp_trouble.c:264-267,404-407) # 2) global/bounce.c:322 status=SOFTBOUNCE # example text: unknown user: "peter" # 3) global/bounce.c:322,512 status=bounced # example text: host host.example.com[2606:2800:220:1:248:1893:25c8:1946] said: 500 need some sleep (in reply to DATA command) # (bounce_append() is called in smtp/smtp_trouble.c:264-267,404-407) # 4) global/sent.c:162 status=sent # example text: 250 2.0.0 Ok: queued as D9A33901180 # (sent() is called in smtp/smtp_rcpt.c:175-177) if msg.startswith('to=<'): _parse_log_adhoc(msg, res) # sets action to 'delivery_status' # smtp/smtp_proto.c:417 # enabling PIX workarounds: %s for %s elif msg.startswith('enabling PIX workarounds: '): res['action'] = 'ignore' # smtp/smtp_connect.c:1057 # smtp/smtp_proto.c:1144-1175 smtp_hbc_logger (hbc = header body checks) # smtp/smtp_proto.c:266-269 smtp_hbc_callbacks # smtp/smtp.c:1326-1333 hbc_header_checks_create,hbc_body_checks_create # global/header_body_checks.c:366-415 return hbc where hbc->call_backs contains the logging callback # the hbc struct is used in :1245-1274(smtp_header_rewrite) and the # smtp main protocol loop smtp_loop (line 2250 calling mime_state_alloc) # by calling hbc_header_checks # global/header_body_checks.c:230-305 hbc_action contains the actual call of the logging callback # here we have these actions (corresponding to some of the ACTIONS in man 5 header_checks): # 255: warning, 259: info, 272: replace, 284: prepend, 290: strip # (see global variable smtp_hbc_actions) # {smtp_hbc_action}: {header_or_body} {content}: {text} # {smtp_hbc_action}: {header_or_body} {content} # -> ignore elif msg.split(': ', 1)[0] in smtp_hbc_actions: res['action'] = 'ignore' elif msg.startswith('host ') or msg.startswith('Protocol error: host '): if msg.startswith('Protocol error: '): msg_ = msg[21:] res['smtp_protocol_error'] = True else: msg_ = msg[5:] sep1 = ' refused to talk to me: ' sep2 = ' said: ' if sep1 in msg: # smtp/smtp_proto.c:375-378,472-475,484-487,493-496 # host {hostname_or_unknown}[{ip_address}] refused to talk to me: {error_detail} host_ip, smtp_error = msg_.split(sep1, 1) res['smtp_error'] = smtp_error _strip_host_ip(host_ip, res, pos='l') res['action'] = 'smtp_error' elif sep2 in msg: # smtp/smtp_proto.c:1940-1944,2013-2017,2037-2041,2084-2088,2111-2115 # host {hostname_or_unknown}[{ip_address}] said: {error_text} (in reply to {command}) # (calls smtp_rcpt_fail or smtp_mesg_fail in smtp/smtp_trouble.c # where vsmtp_fill_dsn fills state->why->reason with the # formatted string, possibly prepending 'Protocol error: ') host_ip, smtp_error = msg_.split(sep2, 1) res['smtp_error'] = smtp_error _strip_host_ip(host_ip, res, pos='l') res['action'] = 'smtp_error' else: res['parsed'] = False # smtp/smtp_connect.c:1057 # smtp/smtp_connect.c:997,195 (call to smtp_connect_addr) # network address conversion failed: %m # -> ignore # smtp/smtp_connect.c:1057 # smtp/smtp_connect.c:974 (call to smtp_tls_policy_cache_query) TODO # smtp/smtp_proto.c:839 # smtp/smtp_sasl_proto.c:173-175 # SASL authentication failed: server %s offered no compatible authentication mechanisms for this type of connection security # -> ignore # smtp/smtp_sasl_glue.c:403-405 # SASL authentication failed; authentication protocol loop with server %s # -> ignore # smtp/smtp_addr.c:216-220 # unable to look up host %s: %s # -> ignore # smtp/smtp_addr.c:236-237 # {host}: host not found # -> ignore # smtp/smtp_addr.c:609-610 # unable to find primary relay for %s # -> ignore # smtp/smtp_addr.c:612-613,677 # mail for %s loops back to myself # -> ignore # smtp/smtp_connect.c:141 # Server configuration error # -> ignore # smtp/smtp_connect.c:195 # network address conversion failed: %m # -> ignore # smtp/smtp_connect.c:308-309 # connect to %s[%s]:%d: %m # -> ignore # smtp/smtp_connect.c:311 # connect to %s[%s]: %m # -> ignore # smtp/smtp_connect.c:782 # all network protocols are disabled # -> ignore # smtp/smtp_connect.c:1085-1086 # server unavailable or unable to receive mail # -> ignore # smtp/smtp_tls_policy.c:168 # client TLS configuration problem # -> ignore # smtp/smtp_tls_policy.c:209 # Temporary lookup error # -> ignore # smtp/smtp_tls_policy.c:861-862 # TLSA lookup error for %s:%u # -> ignore # # dsb_simple, vdsb_simple # calls to smtp/smtp_tls_policy.c:769 dane_incopmat (vdsb_simple in lin 789): TODO # # dsb_update # smtp/smtp_tls_policy.c:714-718 TODO # smtp/smtp_sasl_glue.c:359-363 # SASL authentication failed; cannot authenticate to server %s: %s elif msg.startswith( 'SASL authentication failed; cannot authenticate to server ' ): res['action'] = 'ignore' # smtp/smtp_sasl_glue.c:422-426 # SASL authentication failed; non-empty initial %s challenge from server %s: %s elif msg.startswith('SASL authentication failed; non-empty initial '): res['action'] = 'ignore' # smtp/smtp_sasl_glue.c:433-437 # SASL authentication failed; cannot authenticate to server %s: %s elif msg.startswith( 'SASL authentication failed; cannot authenticate to server ' ): res['action'] = 'ignore' # smtp/smtp_sasl_glue.c:460-464 # SASL authentication failed; server %s said: %s elif ( msg.startswith('SASL authentication failed; server ') and ' said: ' in msg ): res['action'] = 'ignore' # smtp/smtp_sasl_glue.c:342-345 # SASL [CACHED] authentication failed; server %s said: %s # and # smtp/smtp_sasl_proto.c:192-193 smtp_sasl_helo_login calling smtp/smtp_sasl_glue.c:311 smtp_sasl_authenticate # SASL [CACHED] authentication failed; server %s said: %s elif msg.startswith('SASL [CACHED] authentication failed; server '): res['action'] = 'ignore' # # smtp_sess_fail # smtp/smtp_sasl_proto.c:173-175 smtp_sasl_helo_login # SASL authentication failed: server %s offered no compatible authentication mechanisms for this type of connection security elif msg.startswith( 'SASL authentication failed: server ' ) and msg.endswith( ' offered no compatible authentication mechanisms ' 'for this type of connection security' ): res['action'] = 'ignore' # smtp/smtp_sasl_proto.c:192-193 smtp_sasl_helo_login calling smtp/smtp_sasl_glue.c:311 smtp_sasl_authenticate # SASL [CACHED] authentication failed; server %s said: %s # -> already done # smtp/smtp_connect.c:1196 prevents smtp/smtp_trouble.c:224-225 and thus # defer_append or bounce_append get called in lines 264-267, which both # call log_adhoc and are covered above # -> already done # # smtp_bulk_fail # smtp/smtp_trouble.c:435,461 (calling smtp_bulk_fail calling msg_info in line 226) # lost connection with {hostname_or_unknown}[{ip_address}] while {text} # example text: receiving the initial server greeting # example text: performing the EHLO handshake # example text: performing the HELO handshake # example text: performing the LHLO handshake # example text: sending MAIL FROM # example text: sending RCPT TO # example text: receiving the STARTTLS response elif msg.startswith('lost connection with '): res['action'] = 'ignore' # smtp/smtp_trouble.c:443,461 (calling smtp_bulk_fail calling msg_info in line 226) # conversation with {hostname_or_unknown}[{ip_address}] timed out while {text} elif msg.startswith('conversation with '): res['action'] = 'ignore' # smtp/smtp_trouble.c:452,461 (calling smtp_bulk_fail calling msg_info in line 226) # local data error while talking to {hostname_or_unknown}[{ip_address}] elif msg.startswith('local data error while talking to '): res['action'] = 'ignore' # # smtp_site_fail # smtp/smtp_proto.c:388-390 # client TLS configuration problem # -> ignore # smtp/smtp_proto.c:555-558,560-563 # mail for {nexthop} loops back to myself # -> ignore # smtp/smtp_proto.c:803-806 # TLS is required, but host %s refused to start TLS: %s # -> ignore # smtp/smtp_proto.c:819-822 # TLS is required, but was not offered by host %s # -> ignore # smtp/smtp_proto.c:824-826 # TLS is required, but our TLS engine is unavailable # -> ignore # smtp/smtp_proto.c:830-832 # TLS is required, but unavailable # -> ignore # smtp/smtp_proto.c:1124-1126 # Server certificate not trusted # -> ignore # smtp/smtp_proto.c:1129-1131 # Server certificate not verified # -> ignore # smtp/smtp_proto.c:1960-1962 # unexpected server message # -> ignore # # smtp_mesg_fail # smtp/smtp_proto.c:650-654 # SMTPUTF8 is required, but was not offered by host %s # -> ignore # smtp/smtp_proto.c:1386-1388 # %s # -> ignore # smtp/smtp_proto.c:2312-2314 # unreadable mail queue entry # -> ignore # smtp/smtp_proto.c:2388-2392 # message size %lu exceeds size limit %.0f of server %s # -> ignore else: res['action'] = 'ignore' res['parsed'] = False def _parse_bounce(msg, res): """ Parse log messages of the bounce component. """ # Logging is mainly done with msg_info. # bounce/bounce_notify_service.c:202-203,290-291 # bounce/bounce_notify_verp.c:239-240 # bounce/bounce_one_service.c:168-169,244-245 # postmaster non-delivery notification: {queue_id} if msg.startswith('postmaster non-delivery notification: '): _strip_queue_id(msg, res, pos='r', target_name='bounce_id') res['action'] = 'bounce_final' # bounce/bounce_notify_service.c:243-244 # bounce/bounce_notify_verp.c:187-188 # bounce/bounce_one_service.c:206-207 # sender non-delivery notification: {queue_id} elif msg.startswith('sender non-delivery notification: '): _strip_queue_id(msg, res, pos='r', target_name='bounce_id') res['action'] = 'bounce_final' # bounce/bounce_warn_service.c:191-192,274-275 # postmaster delay notification: {queue_id} elif msg.startswith('postmaster delay notification: '): _strip_queue_id(msg, res, pos='r', target_name='bounce_id') res['action'] = 'bounce_delay' # bounce/bounce_warn_service.c:230-231 # sender delay notification: {queue_id} elif msg.startswith('sender delay notification: '): _strip_queue_id(msg, res, pos='r', target_name='bounce_id') res['action'] = 'bounce_delay' # bounce/bounce_trace_service.c:107-108 # not sending trace/success notification for double-bounce message elif ( msg == 'not sending trace/success notification for double-bounce message' ): res['action'] = 'ignore' # bounce/bounce_trace_service.c:115-116 # not sending trace/success notification for single-bounce message elif ( msg == 'not sending trace/success notification for single-bounce message' ): res['action'] = 'ignore' # bounce/bounce_trace_service.c:196-197 # sender delivery status notification: {queue_id} elif msg.startswith('sender delivery status notification: '): _strip_queue_id(msg, res, pos='r', target_name='bounce_id') res['action'] = 'bounce_notify_sender' # bounde/bounce_notify_verp.c:129 # -> ignore # global/bounce.c:368,403 # status=deferred (bounce failed) elif msg == 'status=deferred (bounce failed)': res['action'] = 'bounce_failed' else: res['action'] = 'ignore' res['parsed'] = False def _parse_virtual(msg, res): """ Parse log messages of the virtual component. """ # to=<{to}>, relay=virtual, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) if msg.startswith('to=<'): _parse_log_adhoc(msg, res) # sets action to 'delivery_status' # virtual/unknown.c:61-62 # unknown user: "{user}" # -> ignore # virtual/maildir.c:103 # delivers to maildir # -> ignore # virtual/maildir.c:186-187,211-212 # create maildir file %s: %m # -> ignore # virtual/maildir.c:242 # delivered to maildir # -> ignore # virtual/mailbox.c:101 # delivers to mailbox # -> ignore # virtual/mailbox.c:131,208,234,249,257 # mail system configuration error # -> ignore # virtual/mailbox.c:134-135 # destination %s is not owned by recipient # -> ignore # virtual/mailbox.c:163 # delivered to mailbox # -> ignore else: res['parsed'] = False def _parse_error(msg, res): """ Parse log messages of the error component. """ # to=<{to}>, relay=none, delay={delay}, delays={delays}, dsn={dsn}, status={delivery_status} ({delivery_status_text}) if msg.startswith('to=<'): _parse_log_adhoc(msg, res) # sets action to 'delivery_status' else: res['action'] = 'ignore' res['parsed'] = False def extract_delivery( msg_details: dict, parsed: dict ) -> Tuple[Optional[List[str]], Optional[dict]]: """ Compute a delivery item from a parsed entry. Basically this means that we map the parsed fields to a type ('from' or 'to') and to the database fields for table 'delivery_from' or 'delivery_to'. We branch to functions _extract_{comp} where comp is the name of a Postfix component. Return a list of error strings and a dict with the extracted information. Keys with None values are removed from the dict. """ comp = parsed['comp'] delivery = { 'type': 'to', 't': msg_details['t'], 'queue_id': parsed.get('queue_id'), 'message': msg_details['message'], 'identifier': msg_details['identifier'], 'pid': msg_details['pid'], 'comp': comp, } action = parsed.get('action') if action == 'ignore' or action is None: return None, None elif action == 'subject': delivery['type'] = 'from' delivery['subject'] = parsed.get('subject') elif comp == 'smtpd' or comp.endswith('/smtpd'): delivery = _extract_smtpd(msg_details, parsed, delivery, action) delivery['comp'] = 'smtpd' elif comp == 'qmgr': delivery = _extract_qmgr(msg_details, parsed, delivery, action) elif comp == 'cleanup': delivery = _extract_cleanup(msg_details, parsed, delivery, action) elif comp == 'trivial-rewrite': delivery = _extract_trivial_rewrite( msg_details, parsed, delivery, action ) elif comp in ('smtp', 'lmtp') or comp.endswith('/smtp'): delivery = _extract_smtp(msg_details, parsed, delivery, action) if delivery: delivery['comp'] = 'smtp' elif comp == 'bounce': delivery = _extract_bounce(msg_details, parsed, delivery, action) elif comp == 'virtual': delivery = _extract_virtual(msg_details, parsed, delivery, action) elif comp == 'error': delivery = _extract_error(msg_details, parsed, delivery, action) else: return ['Cannot extract_delivery'], None if 'verp_id' in parsed and delivery and delivery['type'] == 'from': try: delivery['verp_id'] = int(parsed['verp_id']) except Exception: pass # remove keys with None values if delivery: delivery = {k: v for k, v in delivery.items() if v is not None} return None, delivery def _extract_smtpd(msg_details, parsed, delivery, action): delivery['type'] = 'from' if action == 'connect': delivery['host'] = parsed.get('host') delivery['ip'] = parsed.get('ip') delivery['sasl_username'] = parsed.get('sasl_username') delivery['orig_queue_id'] = parsed.get('orig_queue_id') elif action in ('reject', 'hold', 'delay', 'discard', 'redirect'): # Note: Here we may have both sender and recipient. delivery['host'] = parsed.get('host') delivery['ip'] = parsed.get('ip') delivery['status'] = action delivery['sasl_username'] = parsed.get('sasl_username') delivery['sender'] = parsed.get('from') delivery['recipient'] = parsed.get('to') delivery['phase'] = parsed.get('where') delivery['error'] = parsed.get('error') else: return None return delivery def _extract_qmgr(msg_details, parsed, delivery, action): delivery['type'] = 'from' delivery['status'] = action if action == 'expired': delivery['done'] = True elif action == 'removed': delivery['done'] = True elif action == 'queued': delivery['accepted'] = True delivery['sender'] = parsed.get('from') delivery['size'] = parsed.get('size') delivery['nrcpt'] = parsed.get('nrcpt') elif action == 'skipped': delivery['accepted'] = False delivery['done'] = True else: return None return delivery def _extract_cleanup(msg_details, parsed, delivery, action): delivery['type'] = 'from' if action == 'reject': delivery['status'] = action delivery['accepted'] = False if 'mime_error' in parsed: delivery['phase'] = 'mime' delivery['error'] = ( parsed.get('mime_error', '?') + ': ' + parsed.get('mime_error_detail', '?') ) elif action in ('cleanup', 'hold', 'delay', 'discard', 'redirect'): delivery['status'] = action delivery['from'] = parsed.get('from') delivery['to'] = parsed.get('to') elif action == 'message_id': delivery['message_id'] = parsed.get('message_id') elif action == 'resent_message_id': delivery['resent_message_id'] = parsed.get('resent_message_id') elif action == 'milter_action': return None elif action == 'milter_cleanup': return None else: return None return delivery def _extract_trivial_rewrite(msg_details, parsed, delivery, action): return None def _extract_smtp(msg_details, parsed, delivery, action): if action == 'delivery_status': delivery['recipient'] = parsed.get('to') delivery['orig_recipient'] = parsed.get('orig_to') delivery['relay'] = parsed.get('relay') delivery['host'] = parsed.get('host') delivery['destination'] = parsed.get('destination') delivery['port'] = parsed.get('port') delivery['delay'] = parsed.get('delay') delivery['delays'] = parsed.get('delays') delivery['dsn'] = parsed.get('dsn') delivery['status'] = parsed.get('delivery_status') delivery['status_text'] = parsed.get('delivery_status_text') elif action == 'smtp_error': delivery['host'] = parsed.get('host') delivery['destination'] = parsed.get('ip') delivery['status'] = 'smtp_error' delivery['status_text'] = parsed.get('smtp_error') else: return None return delivery def _extract_bounce(msg_details, parsed, delivery, action): if action == 'bounce_delay': delivery['status'] = 'bounce_delay' delivery['status_text'] = parsed.get('bounce_id') elif action == 'bounce_final': delivery['status'] = 'bounce_final' delivery['status_text'] = parsed.get('bounce_id') elif action == 'bounce_notify_sender': delivery['status'] = 'bounce_notify_sender' delivery['status_text'] = parsed.get('bounce_id') elif action == 'bounce_failed': delivery['status'] = 'bounce_failed' else: return None def _extract_virtual(msg_details, parsed, delivery, action): if action == 'delivery_status': delivery['recipient'] = parsed.get('to') delivery['orig_recipient'] = parsed.get('orig_to') delivery['relay'] = parsed.get('relay') delivery['host'] = parsed.get('host') delivery['destination'] = parsed.get('destination') delivery['port'] = parsed.get('port') delivery['delay'] = parsed.get('delay') delivery['delays'] = parsed.get('delays') delivery['dsn'] = parsed.get('dsn') delivery['status'] = parsed.get('delivery_status') delivery['status_text'] = parsed.get('delivery_status_text') else: return None def _extract_error(msg_details, parsed, delivery, action): if action == 'delivery_status': delivery['recipient'] = parsed.get('to') delivery['orig_recipient'] = parsed.get('orig_to') delivery['relay'] = parsed.get('relay') delivery['host'] = parsed.get('host') delivery['destination'] = parsed.get('destination') delivery['port'] = parsed.get('port') delivery['delay'] = parsed.get('delay') delivery['delays'] = parsed.get('delays') delivery['dsn'] = parsed.get('dsn') delivery['status'] = parsed.get('delivery_status') delivery['status_text'] = parsed.get('delivery_status_text') else: return None