Initial Source code version

README.md

@@ -0,0 +1,53 @@
+# Task Queue System – v0.3.0
+
+**FastAPI + PostgreSQL + RabbitMQ** mit produktionsreifen Mail-Prüfungen:
+- MXValidation, SPFValidation, **TLSCheck**, **CertValidity**, **MTASTSCheck**, **TLSRPTCheck**
+- **Task-Abhängigkeiten**: z. B. TLS & CertValidity warten auf MXValidation
+- TLS-Checks mit STARTTLS (Port 25), TLS-Version/Cipher, Zertifikatsdaten & Gültigkeit
+- MTA-STS (Policy-Datei + TXT-Id) und TLS-RPT (rua)
+
+## Quickstart
+```bash
+docker-compose up -d
+
+export DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/tasks"
+export RABBITMQ_URL="amqp://guest:guest@localhost/"
+export RABBITMQ_EXCHANGE="tasks"
+export MAX_RETRIES=3
+export DNS_TIMEOUT=3.0; export DNS_LIFETIME=5.0; export SMTP_TIMEOUT=10.0
+
+# API
+uvicorn app.api:app --host 0.0.0.0 --port 8000 --reload
+
+# Worker
+python -m app.workers.mail_mx_worker
+python -m app.workers.mail_spf_worker
+python -m app.workers.mail_tls_worker
+python -m app.workers.mail_cert_worker
+python -m app.workers.mail_mtasts_worker
+python -m app.workers.mail_tlsrpt_worker
+
+# CLI
+python -m app.cli create-job MailCheck --domain example.com
+```
+
+## Tasks & Dependencies
+```
+MailCheck
+├─ MXValidation (no deps)
+├─ SPFValidation (no deps)
+├─ TLSCheck (depends on: MXValidation)
+├─ CertValidity (depends on: MXValidation)
+├─ MTASTSCheck (no deps)
+└─ TLSRPTCheck (no deps)
+```
+
+## API
+- `POST /jobs` → `{ job_type: "MailCheck", domain: "example.com" }`
+- `GET /jobs/{job_id}`
+
+## Hinweise
+- Für Produktion: Migrationen mit Alembic.
+- MTA-STS-Datei: `https://mta-sts.<domain>/.well-known/mta-sts.txt`
+- TLS nur Port 25 (STARTTLS). 465/587 nicht erforderlich laut Anforderung.
+

app/api.py

@@ -0,0 +1,69 @@
+from fastapi import FastAPI, Depends, HTTPException
+from sqlalchemy.ext.asyncio import AsyncSession
+from app.db import Base, engine, SessionLocal
+from app.schemas import CreateJobRequest, JobInfo, TaskInfo
+from app.services import create_job, get_job_with_tasks
+from app.models import Job, Task
+from typing import List
+from uuid import UUID
+import re, idna
+
+app = FastAPI(title="Task Queue API", version="0.3.0")
+
+async def get_session() -> AsyncSession:
+    async with SessionLocal() as session:
+        yield session
+
+@app.on_event("startup")
+async def on_startup():
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+
+DOMAIN_RE = re.compile(r"^(?=.{1,253}$)(?!-)[A-Za-z0-9-]{1,63}(?<!-)(\.[A-Za-z0-9-]{1,63})+$")
+
+def _validate_domain(domain: str) -> str:
+    try:
+        _ = idna.encode(domain).decode()
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid domain (IDNA)")
+    if not DOMAIN_RE.match(domain):
+        raise HTTPException(status_code=400, detail="Invalid domain format")
+    return domain
+
+@app.post("/jobs", response_model=JobInfo)
+async def create_job_endpoint(req: CreateJobRequest, session: AsyncSession = Depends(get_session)):
+    if not req.job_type:
+        raise HTTPException(status_code=400, detail="job_type is required")
+    domain = _validate_domain(req.domain.strip().lower())
+    job = await create_job(session, req.job_type, domain, req.payload)
+    job = await get_job_with_tasks(session, job.id)
+    return _job_to_schema(job)
+
+@app.get("/jobs/{job_id}", response_model=JobInfo)
+async def get_job_endpoint(job_id: UUID, session: AsyncSession = Depends(get_session)):
+    job = await get_job_with_tasks(session, job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    return _job_to_schema(job)
+
+def _job_to_schema(job: Job) -> JobInfo:
+    tasks: List[TaskInfo] = [
+        TaskInfo(
+            id=t.id,
+            name=t.name,
+            status=t.status.value if hasattr(t.status, "value") else str(t.status),
+            retries=t.retries,
+            max_retries=t.max_retries,
+            result=t.result,
+            error=t.error,
+        )
+        for t in job.tasks
+    ]
+    return JobInfo(
+        id=job.id,
+        job_type=job.job_type,
+        status=job.status.value if hasattr(job.status, "value") else str(job.status),
+        domain=job.domain,
+        payload=job.payload,
+        tasks=tasks,
+    )

app/cli.py

@@ -0,0 +1,83 @@
+
+# app/cli.py
+import argparse
+import sys
+import json
+import os
+import httpx
+
+DEFAULT_API_URL = os.environ.get("TASK_QUEUE_API_URL", "http://127.0.0.1:8000")
+
+def cmd_create_job(args: argparse.Namespace) -> int:
+    """
+    Legt einen neuen Job an (MailCheck) und gibt Status/Tasks aus.
+    """
+    payload = {}  # ggf. später erweitern
+    body = {"job_type": args.job_type, "domain": args.domain, "payload": payload}
+    try:
+        with httpx.Client(base_url=args.api_url, timeout=15) as client:
+            resp = client.post("/jobs", json=body)
+            resp.raise_for_status()
+            job = resp.json()
+    except httpx.HTTPError as e:
+        print(f"[ERROR] API-Request fehlgeschlagen: {e}", file=sys.stderr)
+        return 1
+
+    print(f"Job {job['id']} ({job['job_type']}) Domain: {job['domain']} Status: {job['status']}")
+    for t in job.get("tasks", []):
+        print(f"- Task {t['name']}: {t['status']} (id={t['id']})")
+    return 0
+
+def cmd_get_job(args: argparse.Namespace) -> int:
+    """
+    Holt einen bestehenden Job und zeigt Status/Tasks.
+    """
+    try:
+        with httpx.Client(base_url=args.api_url, timeout=15) as client:
+            resp = client.get(f"/jobs/{args.job_id}")
+            resp.raise_for_status()
+            job = resp.json()
+    except httpx.HTTPError as e:
+        print(f"[ERROR] API-Request fehlgeschlagen: {e}", file=sys.stderr)
+        return 1
+
+    print(json.dumps(job, indent=2))
+    return 0
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="tq",
+        description="Task-Queue CLI (FastAPI + PostgreSQL + RabbitMQ) – RHEL-freundlich mit argparse",
+    )
+    parser.add_argument(
+        "--api-url", default=DEFAULT_API_URL,
+        help=f"Basis-URL der API (Default: {DEFAULT_API_URL})"
+    )
+
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    # create-job
+    p_create = sub.add_parser("create-job", help="Neuen Job anlegen")
+    p_create.add_argument("job_type", help="JobType, z. B. MailCheck")
+    p_create.add_argument("--domain", "-d", required=True, help="Domain für den Job (z. B. example.com)")
+    p_create.set_defaults(func=cmd_create_job)
+
+    # get-job
+    p_get = sub.add_parser("get-job", help="Job-Status abrufen")
+    p_get.add_argument("job_id", help="Job-ID (UUID)")
+    p_get.set_defaults(func=cmd_get_job)
+
+    return parser
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    # Subcommand dispatch
+    if hasattr(args, "func"):
+        return args.func(args)
+    parser.print_help()
+    return 2
+
+if __name__ == "__main__":
+    sys.exit(main())
+

app/config.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+import os
+
+class Settings(BaseModel):
+    DATABASE_URL: str = os.getenv("DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/tasks")
+    RABBITMQ_URL: str = os.getenv("RABBITMQ_URL", "amqp://guest:guest@localhost/")
+    RABBITMQ_EXCHANGE: str = os.getenv("RABBITMQ_EXCHANGE", "tasks")
+    RABBITMQ_EXCHANGE_TYPE: str = "topic"
+    MAX_RETRIES: int = int(os.getenv("MAX_RETRIES", "3"))
+    DNS_TIMEOUT: float = float(os.getenv("DNS_TIMEOUT", "3.0"))
+    DNS_LIFETIME: float = float(os.getenv("DNS_LIFETIME", "5.0"))
+    SMTP_TIMEOUT: float = float(os.getenv("SMTP_TIMEOUT", "10.0"))
+    HTTP_TIMEOUT: float = float(os.getenv("HTTP_TIMEOUT", "8.0"))
+
+settings = Settings()

app/db.py

@@ -0,0 +1,9 @@
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
+from sqlalchemy.orm import DeclarativeBase
+from app.config import settings
+
+engine = create_async_engine(settings.DATABASE_URL, echo=False, future=True)
+SessionLocal = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
+
+class Base(DeclarativeBase):
+    pass

app/job_registry.py

@@ -0,0 +1,22 @@
+from typing import Dict, List, Tuple
+
+# Directed Acyclic Graph (DAG): task -> [dependencies]
+JOB_GRAPH: Dict[str, Dict[str, List[str]]] = {
+    "MailCheck": {
+        "MXValidation": [],
+        "SPFValidation": [],
+        "TLSCheck": ["MXValidation"],
+        "CertValidity": ["MXValidation"],
+        "MTASTSCheck": [],
+        "TLSRPTCheck": [],
+    }
+}
+
+def routing_key(job_type: str, task_name: str) -> str:
+    return f"{job_type}.{task_name}"
+
+def list_tasks_for_job(job_type: str) -> List[str]:
+    return list(JOB_GRAPH.get(job_type, {}).keys())
+
+def list_dependencies(job_type: str, task_name: str) -> List[str]:
+    return JOB_GRAPH.get(job_type, {}).get(task_name, [])

app/models.py

@@ -0,0 +1,54 @@
+import enum, uuid
+from sqlalchemy import String, Enum, ForeignKey, Integer, DateTime, JSON
+from sqlalchemy.orm import relationship, Mapped, mapped_column
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.sql import func
+from app.db import Base
+
+class JobStatus(str, enum.Enum):
+    PENDING = "pending"
+    QUEUED = "queued"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+
+class TaskStatus(str, enum.Enum):
+    PENDING = "pending"
+    QUEUED = "queued"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+
+class Job(Base):
+    __tablename__ = "jobs"
+    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    job_type: Mapped[str] = mapped_column(String(64), index=True)
+    status: Mapped[JobStatus] = mapped_column(Enum(JobStatus), default=JobStatus.PENDING, index=True)
+    domain: Mapped[str] = mapped_column(String(255), index=True)
+    payload: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    created_at: Mapped[str] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[str] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+    tasks: Mapped[list["Task"]] = relationship("Task", back_populates="job", cascade="all, delete-orphan", lazy="selectin")
+
+class Task(Base):
+    __tablename__ = "tasks"
+    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    job_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="CASCADE"), index=True)
+    name: Mapped[str] = mapped_column(String(64), index=True)
+    status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), default=TaskStatus.PENDING, index=True)
+    routing_key: Mapped[str] = mapped_column(String(128), index=True)
+    retries: Mapped[int] = mapped_column(Integer, default=0)
+    max_retries: Mapped[int] = mapped_column(Integer, default=3)
+    worker_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    error: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    created_at: Mapped[str] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    started_at: Mapped[str | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    finished_at: Mapped[str | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    job: Mapped["Job"] = relationship("Job", back_populates="tasks")
+
+class TaskDependency(Base):
+    __tablename__ = "task_dependencies"
+    task_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="CASCADE"), primary_key=True)
+    depends_on_task_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="CASCADE"), primary_key=True)

app/rabbitmq.py

@@ -0,0 +1,27 @@
+import json
+import aio_pika
+from typing import Any
+from app.config import settings
+
+async def get_connection():
+    return await aio_pika.connect_robust(settings.RABBITMQ_URL)
+
+async def publish_task_message(routing_key: str, message: dict[str, Any]):
+    connection = await get_connection()
+    async with connection:
+        channel = await connection.channel()
+        exchange = await channel.declare_exchange(
+            settings.RABBITMQ_EXCHANGE,
+            aio_pika.ExchangeType.TOPIC,
+            durable=True,
+        )
+        body = json.dumps(message).encode("utf-8")
+        await exchange.publish(
+            aio_pika.Message(
+                body=body,
+                content_type="application/json",
+                delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
+                correlation_id=str(message.get("task_id")),
+            ),
+            routing_key=routing_key,
+        )

app/schemas.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List
+from uuid import UUID
+
+class CreateJobRequest(BaseModel):
+  job_type: str = Field(examples=["MailCheck"])
+  domain: str = Field(examples=["example.com"])
+  payload: dict = Field(default_factory=dict)
+
+class TaskInfo(BaseModel):
+  id: UUID
+  name: str
+  status: str
+  retries: int
+  max_retries: int
+  result: Optional[dict] = None
+  error: Optional[str] = None
+
+class JobInfo(BaseModel):
+  id: UUID
+  job_type: str
+  status: str
+  domain: str
+  payload: dict | None
+  tasks: List[TaskInfo]

app/services.py

@@ -0,0 +1,110 @@
+from uuid import UUID
+from sqlalchemy import select, update, func
+from sqlalchemy.ext.asyncio import AsyncSession
+from datetime import datetime
+from app.models import Job, Task, TaskStatus, JobStatus, TaskDependency
+from app.job_registry import list_tasks_for_job, list_dependencies, routing_key
+from app.rabbitmq import publish_task_message
+from app.config import settings
+
+async def create_job(session: AsyncSession, job_type: str, domain: str, payload: dict) -> Job:
+    job = Job(job_type=job_type, domain=domain, payload=payload, status=JobStatus.PENDING)
+    session.add(job)
+    await session.flush()
+
+    # 1) Tasks anlegen
+    name_to_task: dict[str, Task] = {}
+    for name in list_tasks_for_job(job_type):
+        t = Task(
+            job_id=job.id,
+            name=name,
+            status=TaskStatus.PENDING,
+            routing_key=routing_key(job_type, name),
+            max_retries=settings.MAX_RETRIES,
+        )
+        session.add(t)
+        await session.flush()
+        name_to_task[name] = t
+
+    # 2) Dependencies anlegen
+    for name, task in name_to_task.items():
+        for dep_name in list_dependencies(job_type, name):
+            dep_task = name_to_task[dep_name]
+            session.add(TaskDependency(task_id=task.id, depends_on_task_id=dep_task.id))
+
+    await session.commit()
+
+    # 3) Nur Tasks ohne Dependencies veröffentlichen
+    for name, task in name_to_task.items():
+        deps = list_dependencies(job_type, name)
+        if not deps:
+            await _publish_task(session, job, task)
+
+    await refresh_job_status(session, job.id)
+    await session.commit()
+    return job
+
+async def _publish_task(session: AsyncSession, job: Job, task: Task):
+    await publish_task_message(
+        routing_key=task.routing_key,
+        message={
+            "job_id": str(job.id),
+            "task_id": str(task.id),
+            "task_name": task.name,
+            "job_type": job.job_type,
+            "payload": {"domain": job.domain, **(job.payload or {})},
+        },
+    )
+    await session.execute(update(Task).where(Task.id == task.id).values(status=TaskStatus.QUEUED))
+    await session.commit()
+
+async def release_dependent_tasks(session: AsyncSession, completed_task_id: UUID):
+    # Finde Tasks, die von completed_task_id abhängen
+    res = await session.execute(select(TaskDependency.task_id).where(TaskDependency.depends_on_task_id == completed_task_id))
+    candidate_ids = [row[0] for row in res.all()]
+    if not candidate_ids:
+        return
+    # Für jeden Kandidaten prüfen, ob ALLE Dependencies SUCCESS sind
+    for task_id in candidate_ids:
+        # task + job laden
+        res_task = await session.execute(select(Task, Job).join(Job, Task.job_id == Job.id).where(Task.id == task_id))
+        row = res_task.first()
+        if not row:
+            continue
+        task, job = row
+        # Alle deps status
+        res_deps = await session.execute(
+            select(func.count()).select_from(TaskDependency)
+            .join(Task, TaskDependency.depends_on_task_id == Task.id)
+            .where(TaskDependency.task_id == task.id, Task.status != TaskStatus.SUCCESS)
+        )
+        remaining = res_deps.scalar_one()
+        if remaining == 0 and task.status == TaskStatus.PENDING:
+            await _publish_task(session, job, task)
+
+async def get_job(session: AsyncSession, job_id: UUID) -> Job | None:
+    res = await session.execute(select(Job).where(Job.id == job_id))
+    return res.scalar_one_or_none()
+
+async def get_job_with_tasks(session: AsyncSession, job_id: UUID) -> Job | None:
+    return await get_job(session, job_id)
+
+async def refresh_job_status(session: AsyncSession, job_id: UUID) -> None:
+    res = await session.execute(select(Task).where(Task.job_id == job_id))
+    tasks = res.scalars().all()
+    if not tasks:
+        await session.execute(update(Job).where(Job.id == job_id).values(status=JobStatus.PENDING))
+        return
+    if all(t.status == TaskStatus.SUCCESS for t in tasks):
+        new_status = JobStatus.SUCCESS
+    elif any(t.status == TaskStatus.FAILED for t in tasks) and not any(
+        t.status in (TaskStatus.RUNNING, TaskStatus.QUEUED, TaskStatus.PENDING) for t in tasks
+    ):
+        new_status = JobStatus.FAILED
+    elif any(t.status == TaskStatus.RUNNING for t in tasks):
+        new_status = JobStatus.RUNNING
+    elif any(t.status == TaskStatus.QUEUED for t in tasks):
+        new_status = JobStatus.QUEUED
+    else:
+        new_status = JobStatus.PENDING
+    await session.execute(update(Job).where(Job.id == job_id).values(status=new_status, updated_at=datetime.utcnow()))

app/workers/base_worker.py

@@ -0,0 +1,95 @@
+import json
+import asyncio
+from uuid import UUID
+from datetime import datetime
+import aio_pika
+from sqlalchemy import update, select
+from app.db import SessionLocal
+from app.models import Task, TaskStatus
+from app.services import refresh_job_status, release_dependent_tasks
+from app.config import settings
+
+class BaseWorker:
+    def __init__(self, routing_key: str, worker_id: str):
+        self.routing_key = routing_key
+        self.worker_id = worker_id
+
+    async def run(self):
+        connection = await aio_pika.connect_robust(settings.RABBITMQ_URL)
+        async with connection:
+            channel = await connection.channel()
+            await channel.set_qos(prefetch_count=1)
+            exchange = await channel.declare_exchange(
+                settings.RABBITMQ_EXCHANGE, aio_pika.ExchangeType.TOPIC, durable=True
+            )
+            queue = await channel.declare_queue(
+                name=f"q.{self.routing_key}",
+                durable=True,
+                arguments={
+                    "x-dead-letter-exchange": f"{settings.RABBITMQ_EXCHANGE}.dlx"
+                },
+            )
+            await queue.bind(exchange, routing_key=self.routing_key)
+            print(f"[{self.worker_id}] listening on {self.routing_key} ...")
+            await queue.consume(self._on_message, no_ack=False)
+            await asyncio.Future()
+
+    async def _on_message(self, message: aio_pika.IncomingMessage):
+        async with message.process(requeue=False):
+            data = json.loads(message.body.decode("utf-8"))
+            task_id = UUID(data["task_id"])
+            job_id = UUID(data["job_id"])
+            payload = data.get("payload", {})
+            try:
+                await self._set_status(task_id, TaskStatus.RUNNING, started_at=datetime.utcnow())
+                result = await self.process(payload, job_id=job_id, task_id=task_id)
+                await self._set_status(task_id, TaskStatus.SUCCESS, result=result, finished_at=datetime.utcnow())
+                async with SessionLocal() as session:
+                    await release_dependent_tasks(session, task_id)
+                    await refresh_job_status(session, job_id)
+                    await session.commit()
+                print(f"[{self.worker_id}] SUCCESS {task_id}")
+            except Exception as e:
+                print(f"[{self.worker_id}] ERROR: {e}")
+                await self._handle_failure(message, str(e), job_id)
+
+    async def _handle_failure(self, message: aio_pika.IncomingMessage, err: str, job_id):
+        data = json.loads(message.body.decode("utf-8"))
+        task_id = UUID(data["task_id"])
+        async with SessionLocal() as session:
+            res = await session.execute(select(Task).where(Task.id == task_id))
+            task = res.scalar_one_or_none()
+            if task is None:
+                return
+            new_retries = (task.retries or 0) + 1
+            if new_retries <= (task.max_retries or settings.MAX_RETRIES):
+                await session.execute(
+                    update(Task)
+                    .where(Task.id == task_id)
+                    .values(status=TaskStatus.QUEUED, retries=new_retries, error=f"Retry {new_retries}: {err}")
+                )
+                await session.commit()
+                await asyncio.sleep(min(2 ** new_retries, 30))
+                connection = await aio_pika.connect_robust(settings.RABBITMQ_URL)
+                async with connection:
+                    ch = await connection.channel()
+                    ex = await ch.declare_exchange(settings.RABBITMQ_EXCHANGE, aio_pika.ExchangeType.TOPIC, durable=True)
+                    await ex.publish(aio_pika.Message(body=message.body), routing_key=message.routing_key)
+            else:
+                await session.execute(
+                    update(Task)
+                    .where(Task.id == task_id)
+                    .values(status=TaskStatus.FAILED, error=f"Max retries exceeded: {err}", finished_at=datetime.utcnow())
+                )
+                await refresh_job_status(session, job_id)
+                await session.commit()
+
+    async def _set_status(self, task_id: UUID, status: TaskStatus, **extra):
+        async with SessionLocal() as session:
+            await session.execute(
+                update(Task).where(Task.id == task_id).values(status=status, worker_id=self.worker_id, **extra)
+            )
+            await session.commit()
+
+    async def process(self, payload: dict, **meta) -> dict:
+        raise NotImplementedError

app/workers/mail_cert_worker.py

@@ -0,0 +1,70 @@
+from typing import Dict, Any, List
+from sqlalchemy import select
+from app.db import SessionLocal
+from app.models import Task
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx, smtp_starttls_probe, parse_cert_time
+from datetime import datetime, timezone
+
+class CertValidityWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        job_id = meta.get('job_id')
+        if not domain:
+            raise ValueError("payload.domain is required")
+        # Hol MX aus MXValidation
+        mx_hosts: List[str] = []
+        if job_id:
+            async with SessionLocal() as session:
+                res = await session.execute(select(Task).where(Task.job_id == job_id, Task.name == 'MXValidation'))
+                mx_task = res.scalar_one_or_none()
+                if mx_task and mx_task.result and 'mx_records' in mx_task.result:
+                    mx_hosts = [r['exchange'] for r in mx_task.result.get('mx_records', [])]
+        if not mx_hosts:
+            mx = await resolve_mx(domain)
+            mx_hosts = [m.exchange for m in mx]
+        results: List[Dict[str, Any]] = []
+        now = datetime.now(timezone.utc)
+        for host in mx_hosts:
+            probe = await smtp_starttls_probe(host, 25)
+            cert = probe.get('cert')
+            entry: Dict[str, Any] = {'host': host, 'tls_established': probe.get('tls_established', False)}
+            if cert:
+                nb = parse_cert_time(cert.get('notBefore')) if cert.get('notBefore') else None
+                na = parse_cert_time(cert.get('notAfter')) if cert.get('notAfter') else None
+                is_valid = None
+                days_remaining = None
+                if nb and na:
+                    # Hinweis: parse_cert_time gibt naive UTC basierend auf 'GMT'; wir machen sie timezone-aware UTC
+                    nb = nb.replace(tzinfo=timezone.utc)
+                    na = na.replace(tzinfo=timezone.utc)
+                    is_valid = (nb <= now <= na)
+                    days_remaining = int((na - now).total_seconds() // 86400)
+                entry.update({
+                    'not_before': nb.isoformat() if nb else None,
+                    'not_after': na.isoformat() if na else None,
+                    'valid_now': is_valid,
+                    'days_remaining': days_remaining,
+                    'subject': cert.get('subject'),
+                    'issuer': cert.get('issuer'),
+                })
+            else:
+                entry['error'] = probe.get('error') or ('no cert (starttls_advertised=' + str(probe.get('starttls_advertised')) + ')')
+            results.append(entry)
+        return {
+            'certs': results,
+            'summary': {
+                'total_hosts': len(results),
+                'valid_now': sum(1 for r in results if r.get('valid_now')),
+                'expiring_soon_14d': sum(1 for r in results if isinstance(r.get('days_remaining'), int) and r['days_remaining'] <= 14),
+            }
+        }
+
+async def main():
+    worker = CertValidityWorker(routing_key("MailCheck", "CertValidity"), worker_id="cert-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_mtasts_worker.py

@@ -0,0 +1,68 @@
+import httpx
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.config import settings
+from app.workers.utils import resolve_txt
+
+class MTASTSWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get('domain')
+        if not domain:
+            raise ValueError('payload.domain is required')
+        txt_name = f"_mta-sts.{domain}"
+        txts = await resolve_txt(txt_name)
+        txt_policy = None
+        for t in txts:
+            if t.lower().startswith('v=stsv1'):
+                txt_policy = t
+                break
+        # Fetch policy file
+        url = f"https://mta-sts.{domain}/.well-known/mta-sts.txt"
+        policy = {
+            'version': None,
+            'mode': None,
+            'max_age': None,
+            'mx': [],
+        }
+        content = None
+        try:
+            timeout = httpx.Timeout(settings.HTTP_TIMEOUT)
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                resp = await client.get(url)
+                if resp.status_code == 200:
+                    content = resp.text
+        except Exception as e:
+            pass
+        if content:
+            for raw in content.splitlines():
+                line = raw.strip()
+                if not line or line.startswith('#'):
+                    continue
+                if ':' in line:
+                    k, v = [s.strip() for s in line.split(':', 1)]
+                    lk = k.lower()
+                    if lk == 'version':
+                        policy['version'] = v
+                    elif lk == 'mode':
+                        policy['mode'] = v
+                    elif lk == 'max_age':
+                        try:
+                            policy['max_age'] = int(v)
+                        except ValueError:
+                            policy['max_age'] = None
+                    elif lk == 'mx':
+                        policy['mx'].append(v)
+        return {
+            'txt_record': txt_policy,
+            'policy_url': url,
+            'policy': policy,
+            'has_policy': bool(policy['version']),
+        }
+
+async def main():
+    worker = MTASTSWorker(routing_key("MailCheck", "MTASTSCheck"), worker_id="mtasts-worker-1")
+    await worker.run()
+
+if __name__ == '__main__':
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_mx_worker.py

@@ -0,0 +1,24 @@
+import asyncio
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx
+
+class MXValidationWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        if not domain:
+            raise ValueError("payload.domain is required")
+        mx = await resolve_mx(domain)
+        await asyncio.sleep(0)
+        return {
+            "mx_records": [{"exchange": m.exchange, "preference": m.preference} for m in mx],
+            "count": len(mx),
+        }
+
+async def main():
+    worker = MXValidationWorker(routing_key("MailCheck", "MXValidation"), worker_id="mx-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_spf_worker.py

@@ -0,0 +1,22 @@
+import asyncio
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_txt, parse_spf
+
+class SPFValidationWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        if not domain:
+            raise ValueError("payload.domain is required")
+        txts = await resolve_txt(domain)
+        parsed = parse_spf(txts)
+        await asyncio.sleep(0)
+        return parsed
+
+async def main():
+    worker = SPFValidationWorker(routing_key("MailCheck", "SPFValidation"), worker_id="spf-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_tls_worker.py

@@ -0,0 +1,50 @@
+import asyncio
+from typing import Dict, Any, List
+from sqlalchemy import select
+from app.db import SessionLocal
+from app.models import Task
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_mx, smtp_starttls_probe
+
+class TLSCheckWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get("domain")
+        job_id = meta.get('job_id')
+        if not domain:
+            raise ValueError("payload.domain is required")
+        # Versuche MX aus MXValidation-Resultat zu lesen
+        mx_hosts = []
+        if job_id:
+            async with SessionLocal() as session:
+                res = await session.execute(select(Task).where(Task.job_id == job_id, Task.name == 'MXValidation'))
+                mx_task = res.scalar_one_or_none()
+                if mx_task and mx_task.result and 'mx_records' in mx_task.result:
+                    mx_hosts = [r['exchange'] for r in mx_task.result.get('mx_records', [])]
+        if not mx_hosts:
+            mx = await resolve_mx(domain)
+            mx_hosts = [m.exchange for m in mx]
+        if not mx_hosts:
+            return {"mx_hosts": [], "summary": {"total": 0, "starttls_supported": 0, "tls_ok": 0}}
+        results: List[Dict[str, Any]] = []
+        for host in mx_hosts:
+            probe = await smtp_starttls_probe(host, 25)
+            results.append(probe)
+        starttls_supported = sum(1 for r in results if r.get('starttls_advertised'))
+        tls_ok = sum(1 for r in results if r.get('tls_established'))
+        return {
+            "mx_hosts": results,
+            "summary": {
+                "total": len(results),
+                "starttls_supported": starttls_supported,
+                "tls_ok": tls_ok,
+            },
+        }
+
+async def main():
+    worker = TLSCheckWorker(routing_key("MailCheck", "TLSCheck"), worker_id="tls-worker-1")
+    await worker.run()
+
+if __name__ == "__main__":
+    import asyncio as _a
+    _a.run(main())

app/workers/mail_tlsrpt_worker.py

@@ -0,0 +1,37 @@
+from app.workers.base_worker import BaseWorker
+from app.job_registry import routing_key
+from app.workers.utils import resolve_txt
+
+class TLSRPTWorker(BaseWorker):
+    async def process(self, payload: dict, **meta) -> dict:
+        domain = payload.get('domain')
+        if not domain:
+            raise ValueError('payload.domain is required')
+        name = f"_smtp._tls.{domain}"
+        txts = await resolve_txt(name)
+        record = None
+        rua = []
+        for t in txts:
+            tl = t.lower()
+            if tl.startswith('v=tlsrptv1'):
+                record = t
+                # parse rua=mailto:... , mailto:...
+                parts = [p.strip() for p in t.split(';')]
+                for p in parts:
+                    if p.strip().lower().startswith('rua='):
+                        v = p.split('=',1)[1]
+                        rua = [x.strip() for x in v.split(',') if x.strip()]
+                break
+        return {
+            'txt_record': record,
+            'rua': rua,
+            'enabled': record is not None,
+        }
+
+async def main():
+    worker = TLSRPTWorker(routing_key("MailCheck", "TLSRPTCheck"), worker_id="tlsrpt-worker-1")
+    await worker.run()
+
+if __name__ == '__main__':
+    import asyncio as _a
+    _a.run(main())

app/workers/utils.py

@@ -0,0 +1,116 @@
+import asyncio
+import socket
+import ssl
+import smtplib
+from dataclasses import dataclass
+from typing import List, Dict, Any
+import dns.resolver
+from app.config import settings
+from datetime import datetime
+
+@dataclass
+class MXRecord:
+    preference: int
+    exchange: str
+
+async def resolve_mx(domain: str) -> List[MXRecord]:
+    def _sync() -> List[MXRecord]:
+        r = dns.resolver.Resolver()
+        r.timeout = settings.DNS_TIMEOUT
+        r.lifetime = settings.DNS_LIFETIME
+        answers = r.resolve(domain, 'MX')
+        recs = []
+        for rr in answers:
+            recs.append(MXRecord(preference=int(rr.preference), exchange=str(rr.exchange).rstrip('.')))
+        recs.sort(key=lambda x: x.preference)
+        return recs
+    return await asyncio.to_thread(_sync)
+
+async def resolve_txt(domain: str) -> List[str]:
+    def _sync() -> List[str]:
+        r = dns.resolver.Resolver()
+        r.timeout = settings.DNS_TIMEOUT
+        r.lifetime = settings.DNS_LIFETIME
+        try:
+            answers = r.resolve(domain, 'TXT')
+        except Exception:
+            return []
+        out = []
+        for rr in answers:
+            s = ''.join(part.decode('utf-8') if isinstance(part, bytes) else str(part) for part in getattr(rr, 'strings', []))
+            if not s:
+                s = str(rr)
+            s = s.strip('"')
+            out.append(s)
+        return out
+    return await asyncio.to_thread(_sync)
+
+def parse_spf(txt_records: List[str]) -> Dict[str, Any]:
+    spf_records = [t for t in txt_records if t.lower().startswith('v=spf1')]
+    details = []
+    for spf in spf_records:
+        tokens = spf.split()
+        mechanisms = [t for t in tokens[1:]]
+        qualifier = next((t for t in tokens if t.endswith('all')), None)
+        details.append({
+            'record': spf,
+            'mechanisms': mechanisms,
+            'has_all': any(tok.endswith('all') for tok in tokens),
+            'all_qualifier': qualifier,
+        })
+    return {
+        'present': len(spf_records) > 0,
+        'records': spf_records,
+        'parsed': details,
+    }
+
+async def smtp_starttls_probe(host: str, port: int = 25) -> Dict[str, Any]:
+    def _sync() -> Dict[str, Any]:
+        context = ssl.create_default_context()
+        context.check_hostname = True
+        context.verify_mode = ssl.CERT_REQUIRED
+        result: Dict[str, Any] = {
+            'host': host,
+            'port': port,
+            'ehlo': False,
+            'starttls_advertised': False,
+            'tls_established': False,
+            'tls_version': None,
+            'cipher': None,
+            'cert': None,
+        }
+        try:
+            with smtplib.SMTP(host=host, port=port, timeout=settings.SMTP_TIMEOUT) as server:
+                code, _ = server.ehlo()
+                result['ehlo'] = (200 <= code < 400)
+                if 'starttls' in (server.esmtp_features or {}):
+                    result['starttls_advertised'] = True
+                    code, _ = server.starttls(context=context)
+                    if 200 <= code < 400:
+                        result['tls_established'] = True
+                        server.ehlo()
+                        try:
+                            sslobj = server.sock
+                            result['tls_version'] = getattr(sslobj, 'version', lambda: None)()
+                            result['cipher'] = getattr(sslobj, 'cipher', lambda: None)()
+                            try:
+                                cert = sslobj.getpeercert()
+                                result['cert'] = cert
+                            except Exception:
+                                pass
+                        except Exception:
+                            pass
+                else:
+                    result['starttls_advertised'] = False
+        except (ssl.SSLError, smtplib.SMTPException, OSError, socket.error) as e:
+            result['error'] = str(e)
+        return result
+    return await asyncio.to_thread(_sync)
+
+# Cert date parsing (OpenSSL style: 'Jun  5 12:00:00 2025 GMT')
+def parse_cert_time(s: str) -> datetime | None:
+    try:
+        norm = ' '.join(s.split())
+        return datetime.strptime(norm, '%b %d %H:%M:%S %Y %Z')
+    except Exception:
+        return None

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3.9"
+services:
+  postgres:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: tasks
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+
+  rabbitmq:
+    image: rabbitmq:3-management
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+
+volumes:
+  pgdata:

pyproject.toml

@@ -0,0 +1,35 @@
+
+[tool.poetry]
+name = "task-queue-system"
+version = "0.3.0"
+description = "Async task queue with FastAPI, PostgreSQL, RabbitMQ; MailCheck workers incl. MX, SPF, TLS, CertValidity, MTA-STS, TLS-RPT with dependencies"
+authors = ["Frank Agerholm <you@example.com>"]
+readme = "README.md"
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = ">=3.12,<3.13"
+fastapi = "^0.115"
+uvicorn = {version = "^0.30", extras = ["standard"]}
+sqlalchemy = "^2.0"
+asyncpg = "^0.29"
+pydantic = "^2.6"
+aio-pika = "^9.4"
+httpx = "^0.27"
+python-dotenv = "^1.0"
+dnspython = "^2.6"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.10"
+ruff = "^0.6"
+
+[tool.poetry.scripts]
+# optional: CLI entrypoint (statt python -m app.cli)
+tq = "app.cli:main"
+
+[tool.poetry.urls]
+"Homepage" = "https://example.com"
+
+[build-system]
+requires = ["poetry-core>=1.6.0"]
+