Initial Source code version

app/workers/base_worker.py

@@ -0,0 +1,95 @@
+import json
+import asyncio
+from uuid import UUID
+from datetime import datetime
+import aio_pika
+from sqlalchemy import update, select
+from app.db import SessionLocal
+from app.models import Task, TaskStatus
+from app.services import refresh_job_status, release_dependent_tasks
+from app.config import settings
+
+class BaseWorker:
+    def __init__(self, routing_key: str, worker_id: str):
+        self.routing_key = routing_key
+        self.worker_id = worker_id
+
+    async def run(self):
+        connection = await aio_pika.connect_robust(settings.RABBITMQ_URL)
+        async with connection:
+            channel = await connection.channel()
+            await channel.set_qos(prefetch_count=1)
+            exchange = await channel.declare_exchange(
+                settings.RABBITMQ_EXCHANGE, aio_pika.ExchangeType.TOPIC, durable=True
+            )
+            queue = await channel.declare_queue(
+                name=f"q.{self.routing_key}",
+                durable=True,
+                arguments={
+                    "x-dead-letter-exchange": f"{settings.RABBITMQ_EXCHANGE}.dlx"
+                },
+            )
+            await queue.bind(exchange, routing_key=self.routing_key)
+            print(f"[{self.worker_id}] listening on {self.routing_key} ...")
+            await queue.consume(self._on_message, no_ack=False)
+            await asyncio.Future()
+
+    async def _on_message(self, message: aio_pika.IncomingMessage):
+        async with message.process(requeue=False):
+            data = json.loads(message.body.decode("utf-8"))
+            task_id = UUID(data["task_id"])
+            job_id = UUID(data["job_id"])
+            payload = data.get("payload", {})
+            try:
+                await self._set_status(task_id, TaskStatus.RUNNING, started_at=datetime.utcnow())
+                result = await self.process(payload, job_id=job_id, task_id=task_id)
+                await self._set_status(task_id, TaskStatus.SUCCESS, result=result, finished_at=datetime.utcnow())
+                async with SessionLocal() as session:
+                    await release_dependent_tasks(session, task_id)
+                    await refresh_job_status(session, job_id)
+                    await session.commit()
+                print(f"[{self.worker_id}] SUCCESS {task_id}")
+            except Exception as e:
+                print(f"[{self.worker_id}] ERROR: {e}")
+                await self._handle_failure(message, str(e), job_id)
+
+    async def _handle_failure(self, message: aio_pika.IncomingMessage, err: str, job_id):
+        data = json.loads(message.body.decode("utf-8"))
+        task_id = UUID(data["task_id"])
+        async with SessionLocal() as session:
+            res = await session.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one_or_none()
+            if task is None:
+                return
+            new_retries = (task.retries or 0) + 1
+            if new_retries <= (task.max_retries or settings.MAX_RETRIES):
+                await session.execute(
+                    update(Task)
+                    .where(Task.id == task_id)
+                    .values(status=TaskStatus.QUEUED, retries=new_retries, error=f"Retry {new_retries}: {err}")
+                )
+                await session.commit()
+                await asyncio.sleep(min(2 ** new_retries, 30))
+                connection = await aio_pika.connect_robust(settings.RABBITMQ_URL)
+                async with connection:
+                    ch = await connection.channel()
+                    ex = await ch.declare_exchange(settings.RABBITMQ_EXCHANGE, aio_pika.ExchangeType.TOPIC, durable=True)
+                    await ex.publish(aio_pika.Message(body=message.body), routing_key=message.routing_key)
+            else:
+                await session.execute(
+                    update(Task)
+                    .where(Task.id == task_id)
+                    .values(status=TaskStatus.FAILED, error=f"Max retries exceeded: {err}", finished_at=datetime.utcnow())
+                )
+                await refresh_job_status(session, job_id)
+                await session.commit()
+
+    async def _set_status(self, task_id: UUID, status: TaskStatus, **extra):
+        async with SessionLocal() as session:
+            await session.execute(
+                update(Task).where(Task.id == task_id).values(status=status, worker_id=self.worker_id, **extra)
+            )
+            await session.commit()
+
+    async def process(self, payload: dict, **meta) -> dict:
+        raise NotImplementedError

app/workers/mail_cert_worker.py

@@ -0,0 +1,70 @@
+from typing import Dict, Any, List
+from sqlalchemy import select
+from app.db import SessionLocal
+from app.models import Task
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx, smtp_starttls_probe, parse_cert_time
+from datetime import datetime, timezone
+
+class CertValidityWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        job_id = meta.get('job_id')
+        if not domain:
+            raise ValueError("payload.domain is required")
+        # Hol MX aus MXValidation
+        mx_hosts: List[str] = []
+        if job_id:
+            async with SessionLocal() as session:
+                res = await session.execute(select(Task).where(Task.job_id == job_id, Task.name == 'MXValidation'))
+                mx_task = res.scalar_one_or_none()
+                if mx_task and mx_task.result and 'mx_records' in mx_task.result:
+                    mx_hosts = [r['exchange'] for r in mx_task.result.get('mx_records', [])]
+        if not mx_hosts:
+            mx = await resolve_mx(domain)
+            mx_hosts = [m.exchange for m in mx]
+        results: List[Dict[str, Any]] = []
+        now = datetime.now(timezone.utc)
+        for host in mx_hosts:
+            probe = await smtp_starttls_probe(host, 25)
+            cert = probe.get('cert')
+            entry: Dict[str, Any] = {'host': host, 'tls_established': probe.get('tls_established', False)}
+            if cert:
+                nb = parse_cert_time(cert.get('notBefore')) if cert.get('notBefore') else None
+                na = parse_cert_time(cert.get('notAfter')) if cert.get('notAfter') else None
+                is_valid = None
+                days_remaining = None
+                if nb and na:
+                    # Hinweis: parse_cert_time gibt naive UTC basierend auf 'GMT'; wir machen sie timezone-aware UTC
+                    nb = nb.replace(tzinfo=timezone.utc)
+                    na = na.replace(tzinfo=timezone.utc)
+                    is_valid = (nb <= now <= na)
+                    days_remaining = int((na - now).total_seconds() // 86400)
+                entry.update({
+                    'not_before': nb.isoformat() if nb else None,
+                    'not_after': na.isoformat() if na else None,
+                    'valid_now': is_valid,
+                    'days_remaining': days_remaining,
+                    'subject': cert.get('subject'),
+                    'issuer': cert.get('issuer'),
+                })
+            else:
+                entry['error'] = probe.get('error') or ('no cert (starttls_advertised=' + str(probe.get('starttls_advertised')) + ')')
+            results.append(entry)
+        return {
+            'certs': results,
+            'summary': {
+                'total_hosts': len(results),
+                'valid_now': sum(1 for r in results if r.get('valid_now')),
+                'expiring_soon_14d': sum(1 for r in results if isinstance(r.get('days_remaining'), int) and r['days_remaining'] <= 14),
+            }
+        }
+
+async def main():
+    worker = CertValidityWorker(routing_key("MailCheck", "CertValidity"), worker_id="cert-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_mtasts_worker.py

@@ -0,0 +1,68 @@
+import httpx
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.config import settings
+from app.workers.utils import resolve_txt
+
+class MTASTSWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get('domain')
+        if not domain:
+            raise ValueError('payload.domain is required')
+        txt_name = f"_mta-sts.{domain}"
+        txts = await resolve_txt(txt_name)
+        txt_policy = None
+        for t in txts:
+            if t.lower().startswith('v=stsv1'):
+                txt_policy = t
+                break
+        # Fetch policy file
+        url = f"https://mta-sts.{domain}/.well-known/mta-sts.txt"
+        policy = {
+            'version': None,
+            'mode': None,
+            'max_age': None,
+            'mx': [],
+        }
+        content = None
+        try:
+            timeout = httpx.Timeout(settings.HTTP_TIMEOUT)
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                resp = await client.get(url)
+                if resp.status_code == 200:
+                    content = resp.text
+        except Exception as e:
+            pass
+        if content:
+            for raw in content.splitlines():
+                line = raw.strip()
+                if not line or line.startswith('#'):
+                    continue
+                if ':' in line:
+                    k, v = [s.strip() for s in line.split(':', 1)]
+                    lk = k.lower()
+                    if lk == 'version':
+                        policy['version'] = v
+                    elif lk == 'mode':
+                        policy['mode'] = v
+                    elif lk == 'max_age':
+                        try:
+                            policy['max_age'] = int(v)
+                        except ValueError:
+                            policy['max_age'] = None
+                    elif lk == 'mx':
+                        policy['mx'].append(v)
+        return {
+            'txt_record': txt_policy,
+            'policy_url': url,
+            'policy': policy,
+            'has_policy': bool(policy['version']),
+        }
+
+async def main():
+    worker = MTASTSWorker(routing_key("MailCheck", "MTASTSCheck"), worker_id="mtasts-worker-1")
+    await worker.run()
+
+if __name__ == '__main__':
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_mx_worker.py

@@ -0,0 +1,24 @@
+import asyncio
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx
+
+class MXValidationWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        if not domain:
+            raise ValueError("payload.domain is required")
+        mx = await resolve_mx(domain)
+        await asyncio.sleep(0)
+        return {
+            "mx_records": [{"exchange": m.exchange, "preference": m.preference} for m in mx],
+            "count": len(mx),
+        }
+
+async def main():
+    worker = MXValidationWorker(routing_key("MailCheck", "MXValidation"), worker_id="mx-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_spf_worker.py

@@ -0,0 +1,22 @@
+import asyncio
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_txt, parse_spf
+
+class SPFValidationWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        if not domain:
+            raise ValueError("payload.domain is required")
+        txts = await resolve_txt(domain)
+        parsed = parse_spf(txts)
+        await asyncio.sleep(0)
+        return parsed
+
+async def main():
+    worker = SPFValidationWorker(routing_key("MailCheck", "SPFValidation"), worker_id="spf-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_tls_worker.py

@@ -0,0 +1,50 @@
+import asyncio
+from typing import Dict, Any, List
+from sqlalchemy import select
+from app.db import SessionLocal
+from app.models import Task
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx, smtp_starttls_probe
+
+class TLSCheckWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        job_id = meta.get('job_id')
+        if not domain:
+            raise ValueError("payload.domain is required")
+        # Versuche MX aus MXValidation-Resultat zu lesen
+        mx_hosts = []
+        if job_id:
+            async with SessionLocal() as session:
+                res = await session.execute(select(Task).where(Task.job_id == job_id, Task.name == 'MXValidation'))
+                mx_task = res.scalar_one_or_none()
+                if mx_task and mx_task.result and 'mx_records' in mx_task.result:
+                    mx_hosts = [r['exchange'] for r in mx_task.result.get('mx_records', [])]
+        if not mx_hosts:
+            mx = await resolve_mx(domain)
+            mx_hosts = [m.exchange for m in mx]
+        if not mx_hosts:
+            return {"mx_hosts": [], "summary": {"total": 0, "starttls_supported": 0, "tls_ok": 0}}
+        results: List[Dict[str, Any]] = []
+        for host in mx_hosts:
+            probe = await smtp_starttls_probe(host, 25)
+            results.append(probe)
+        starttls_supported = sum(1 for r in results if r.get('starttls_advertised'))
+        tls_ok = sum(1 for r in results if r.get('tls_established'))
+        return {
+            "mx_hosts": results,
+            "summary": {
+                "total": len(results),
+                "starttls_supported": starttls_supported,
+                "tls_ok": tls_ok,
+            },
+        }
+
+async def main():
+    worker = TLSCheckWorker(routing_key("MailCheck", "TLSCheck"), worker_id="tls-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_tlsrpt_worker.py

@@ -0,0 +1,37 @@
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_txt
+
+class TLSRPTWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get('domain')
+        if not domain:
+            raise ValueError('payload.domain is required')
+        name = f"_smtp._tls.{domain}"
+        txts = await resolve_txt(name)
+        record = None
+        rua = []
+        for t in txts:
+            tl = t.lower()
+            if tl.startswith('v=tlsrptv1'):
+                record = t
+                # parse rua=mailto:... , mailto:...
+                parts = [p.strip() for p in t.split(';')]
+                for p in parts:
+                    if p.strip().lower().startswith('rua='):
+                        v = p.split('=',1)[1]
+                        rua = [x.strip() for x in v.split(',') if x.strip()]
+                break
+        return {
+            'txt_record': record,
+            'rua': rua,
+            'enabled': record is not None,
+        }
+
+async def main():
+    worker = TLSRPTWorker(routing_key("MailCheck", "TLSRPTCheck"), worker_id="tlsrpt-worker-1")
+    await worker.run()
+
+if __name__ == '__main__':
+    import asyncio as _a
+    _a.run(main())

app/workers/utils.py

@@ -0,0 +1,116 @@
+import asyncio
+import socket
+import ssl
+import smtplib
+from dataclasses import dataclass
+from typing import List, Dict, Any
+import dns.resolver
+from app.config import settings
+from datetime import datetime
+
+@dataclass
+class MXRecord:
+    preference: int
+    exchange: str
+
+async def resolve_mx(domain: str) -> List[MXRecord]:
+    def _sync() -> List[MXRecord]:
+        r = dns.resolver.Resolver()
+        r.timeout = settings.DNS_TIMEOUT
+        r.lifetime = settings.DNS_LIFETIME
+        answers = r.resolve(domain, 'MX')
+        recs = []
+        for rr in answers:
+            recs.append(MXRecord(preference=int(rr.preference), exchange=str(rr.exchange).rstrip('.')))
+        recs.sort(key=lambda x: x.preference)
+        return recs
+    return await asyncio.to_thread(_sync)
+
+async def resolve_txt(domain: str) -> List[str]:
+    def _sync() -> List[str]:
+        r = dns.resolver.Resolver()
+        r.timeout = settings.DNS_TIMEOUT
+        r.lifetime = settings.DNS_LIFETIME
+        try:
+            answers = r.resolve(domain, 'TXT')
+        except Exception:
+            return []
+        out = []
+        for rr in answers:
+            s = ''.join(part.decode('utf-8') if isinstance(part, bytes) else str(part) for part in getattr(rr, 'strings', []))
+            if not s:
+                s = str(rr)
+            s = s.strip('"')
+            out.append(s)
+        return out
+    return await asyncio.to_thread(_sync)
+
+def parse_spf(txt_records: List[str]) -> Dict[str, Any]:
+    spf_records = [t for t in txt_records if t.lower().startswith('v=spf1')]
+    details = []
+    for spf in spf_records:
+        tokens = spf.split()
+        mechanisms = [t for t in tokens[1:]]
+        qualifier = next((t for t in tokens if t.endswith('all')), None)
+        details.append({
+            'record': spf,
+            'mechanisms': mechanisms,
+            'has_all': any(tok.endswith('all') for tok in tokens),
+            'all_qualifier': qualifier,
+        })
+    return {
+        'present': len(spf_records) > 0,
+        'records': spf_records,
+        'parsed': details,
+    }
+
+async def smtp_starttls_probe(host: str, port: int = 25) -> Dict[str, Any]:
+    def _sync() -> Dict[str, Any]:
+        context = ssl.create_default_context()
+        context.check_hostname = True
+        context.verify_mode = ssl.CERT_REQUIRED
+        result: Dict[str, Any] = {
+            'host': host,
+            'port': port,
+            'ehlo': False,
+            'starttls_advertised': False,
+            'tls_established': False,
+            'tls_version': None,
+            'cipher': None,
+            'cert': None,
+        }
+        try:
+            with smtplib.SMTP(host=host, port=port, timeout=settings.SMTP_TIMEOUT) as server:
+                code, _ = server.ehlo()
+                result['ehlo'] = (200 <= code < 400)
+                if 'starttls' in (server.esmtp_features or {}):
+                    result['starttls_advertised'] = True
+                    code, _ = server.starttls(context=context)
+                    if 200 <= code < 400:
+                        result['tls_established'] = True
+                        server.ehlo()
+                        try:
+                            sslobj = server.sock
+                            result['tls_version'] = getattr(sslobj, 'version', lambda: None)()
+                            result['cipher'] = getattr(sslobj, 'cipher', lambda: None)()
+                            try:
+                                cert = sslobj.getpeercert()
+                                result['cert'] = cert
+                            except Exception:
+                                pass
+                        except Exception:
+                            pass
+                else:
+                    result['starttls_advertised'] = False
+        except (ssl.SSLError, smtplib.SMTPException, OSError, socket.error) as e:
+            result['error'] = str(e)
+        return result
+    return await asyncio.to_thread(_sync)
+
+# Cert date parsing (OpenSSL style: 'Jun  5 12:00:00 2025 GMT')
+def parse_cert_time(s: str) -> datetime | None:
+    try:
+        norm = ' '.join(s.split())
+        return datetime.strptime(norm, '%b %d %H:%M:%S %Y %Z')
+    except Exception:
+        return None