timeouts fuer dav gesetzt
This commit is contained in:
@@ -10,3 +10,7 @@ CALDAV_URL=https://dav.mailbox.org/caldav/Y2FsOi8vMC8zMg
|
|||||||
CALDAV_USERNAME=minitux@mailbox.org
|
CALDAV_USERNAME=minitux@mailbox.org
|
||||||
CALDAV_PASSWORD=4711Cayenne64
|
CALDAV_PASSWORD=4711Cayenne64
|
||||||
|
|
||||||
|
CALDAV_TIMEOUT=90
|
||||||
|
CALDAV_RETRIES=3
|
||||||
|
CALDAV_RETRY_BACKOFF=5
|
||||||
|
|
||||||
|
|||||||
@@ -9,3 +9,8 @@ IMAP_MARK_AS_READ=false
|
|||||||
CALDAV_URL=https://dav.mailbox.org/caldav/IHR_KALENDER_ID
|
CALDAV_URL=https://dav.mailbox.org/caldav/IHR_KALENDER_ID
|
||||||
CALDAV_USERNAME=ihr-name@mailbox.org
|
CALDAV_USERNAME=ihr-name@mailbox.org
|
||||||
CALDAV_PASSWORD=IHR_PASSWORT_ODER_APP_PASSWORT
|
CALDAV_PASSWORD=IHR_PASSWORT_ODER_APP_PASSWORT
|
||||||
|
|
||||||
|
Optionale .env-Werte:
|
||||||
|
CALDAV_TIMEOUT=90
|
||||||
|
CALDAV_RETRIES=3
|
||||||
|
CALDAV_RETRY_BACKOFF=5
|
||||||
+114
-29
@@ -1,15 +1,19 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
ics_mail_importer_env_v5.py
|
ics_mail_importer_env_v6.py
|
||||||
---------------------------
|
---------------------------
|
||||||
ICS-Importer für mailbox.org / CalDAV.
|
ICS-Importer für mailbox.org / CalDAV.
|
||||||
|
|
||||||
Verbesserungen in v4:
|
Verbesserungen in v6:
|
||||||
- Liest Mails per BODY.PEEK[] ohne automatisches Setzen von \Seen
|
- CAL-4121-Konflikte werden als "bereits aktueller Serverstand" behandelt
|
||||||
- Markiert nur verarbeitete ICS-Mails optional als gelesen
|
- Timeout-Fehler beim CalDAV-Upload werden mit Retries und Backoff erneut versucht
|
||||||
- Behandelt mailbox.org / OX-Konflikte vom Typ CAL-4121 (neuere Version existiert)
|
- Netzwerk-/Timeout-Probleme werden getrennt von fachlichen Importfehlern protokolliert
|
||||||
als nicht-fatale Konflikte statt als Importfehler
|
- Mails werden nur bei erfolgreicher Verarbeitung oder erkanntem Konflikt als verarbeitet markiert
|
||||||
- Protokolliert Konflikte klar als "bereits neuer vorhanden"
|
|
||||||
|
Optionale .env-Werte:
|
||||||
|
- CALDAV_TIMEOUT=90
|
||||||
|
- CALDAV_RETRIES=3
|
||||||
|
- CALDAV_RETRY_BACKOFF=5
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import imaplib
|
import imaplib
|
||||||
@@ -17,6 +21,7 @@ import email
|
|||||||
import logging
|
import logging
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
@@ -53,36 +58,105 @@ def env_bool(name: str, default: str = "false") -> bool:
|
|||||||
return os.getenv(name, default).strip().lower() in {"1", "true", "yes", "on"}
|
return os.getenv(name, default).strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def env_int(name: str, default: int) -> int:
|
||||||
|
value = os.getenv(name)
|
||||||
|
if value is None or not value.strip():
|
||||||
|
return default
|
||||||
|
return int(value.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def exception_text(exc: Exception) -> str:
|
||||||
|
return str(exc or "")
|
||||||
|
|
||||||
|
|
||||||
def is_newer_version_conflict(exc: Exception) -> bool:
|
def is_newer_version_conflict(exc: Exception) -> bool:
|
||||||
message = str(exc or "").lower()
|
message = exception_text(exc).lower()
|
||||||
patterns = [
|
patterns = [
|
||||||
"412 precondition failed",
|
"412 precondition failed",
|
||||||
"cal-4121",
|
"cal-4121",
|
||||||
"newer version of the appointment already exists",
|
"newer version of the appointment already exists",
|
||||||
"concurrent modification",
|
"concurrent modification",
|
||||||
"client sequence",
|
|
||||||
"actual sequence",
|
"actual sequence",
|
||||||
|
"client sequence",
|
||||||
]
|
]
|
||||||
matches = sum(1 for p in patterns if p in message)
|
return sum(1 for pattern in patterns if pattern in message) >= 2
|
||||||
return matches >= 2
|
|
||||||
|
|
||||||
|
def is_timeout_error(exc: Exception) -> bool:
|
||||||
|
message = exception_text(exc).lower()
|
||||||
|
timeout_patterns = [
|
||||||
|
"read timed out",
|
||||||
|
"timeout",
|
||||||
|
"timed out",
|
||||||
|
"read timeout",
|
||||||
|
]
|
||||||
|
return any(pattern in message for pattern in timeout_patterns)
|
||||||
|
|
||||||
|
|
||||||
|
def add_event_with_retries(calendar, ical_text: str, summary: str, uid: str, sequence, timeout: int, retries: int, backoff: int):
|
||||||
|
last_exception = None
|
||||||
|
|
||||||
|
for attempt in range(1, retries + 1):
|
||||||
|
try:
|
||||||
|
calendar.add_event(ical_text)
|
||||||
|
log.info(f" ✓ Termin importiert: {summary} [{uid[:20]}]")
|
||||||
|
return "imported"
|
||||||
|
except Exception as e:
|
||||||
|
last_exception = e
|
||||||
|
exc_name = type(e).__name__
|
||||||
|
exc_message = exception_text(e)
|
||||||
|
log.info(f" Exception-Typ beim Import: {exc_name}")
|
||||||
|
|
||||||
|
if is_newer_version_conflict(e):
|
||||||
|
log.info(
|
||||||
|
f" ↷ Konflikt ignoriert: Neuere Version existiert bereits: {summary} "
|
||||||
|
f"[{uid[:20]}], SEQUENCE={sequence}"
|
||||||
|
)
|
||||||
|
return "conflict"
|
||||||
|
|
||||||
|
if is_timeout_error(e):
|
||||||
|
if attempt < retries:
|
||||||
|
wait_seconds = backoff * attempt
|
||||||
|
log.warning(
|
||||||
|
f" ⏳ Timeout beim Import von {uid[:20]} (Versuch {attempt}/{retries}, Timeout={timeout}s). "
|
||||||
|
f"Neuer Versuch in {wait_seconds}s ..."
|
||||||
|
)
|
||||||
|
time.sleep(wait_seconds)
|
||||||
|
continue
|
||||||
|
log.warning(
|
||||||
|
f" ✗ Timeout beim Import von {uid[:20]} nach {retries} Versuchen: {exc_message}"
|
||||||
|
)
|
||||||
|
return "timeout"
|
||||||
|
|
||||||
|
log.warning(f" ✗ Fehler beim Import von {uid[:20]}: {exc_message}")
|
||||||
|
return "error"
|
||||||
|
|
||||||
|
if last_exception is not None:
|
||||||
|
raise last_exception
|
||||||
|
return "error"
|
||||||
|
|
||||||
|
|
||||||
def import_ics_to_caldav(ics_bytes: bytes, uid_hash: str):
|
def import_ics_to_caldav(ics_bytes: bytes, uid_hash: str):
|
||||||
caldav_url = os.getenv("CALDAV_URL")
|
caldav_url = os.getenv("CALDAV_URL")
|
||||||
username = os.getenv("CALDAV_USERNAME")
|
username = os.getenv("CALDAV_USERNAME")
|
||||||
password = os.getenv("CALDAV_PASSWORD")
|
password = os.getenv("CALDAV_PASSWORD")
|
||||||
|
timeout = env_int("CALDAV_TIMEOUT", 90)
|
||||||
|
retries = env_int("CALDAV_RETRIES", 3)
|
||||||
|
backoff = env_int("CALDAV_RETRY_BACKOFF", 5)
|
||||||
|
|
||||||
if not caldav_url or not username or not password:
|
if not caldav_url or not username or not password:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"CALDAV_URL, CALDAV_USERNAME oder CALDAV_PASSWORD fehlt in .env"
|
"CALDAV_URL, CALDAV_USERNAME oder CALDAV_PASSWORD fehlt in .env"
|
||||||
)
|
)
|
||||||
|
|
||||||
client = caldav.DAVClient(url=caldav_url, username=username, password=password)
|
client = caldav.DAVClient(url=caldav_url, username=username, password=password, timeout=timeout)
|
||||||
calendar = client.calendar(url=caldav_url)
|
calendar = client.calendar(url=caldav_url)
|
||||||
cal = Calendar.from_ical(ics_bytes)
|
cal = Calendar.from_ical(ics_bytes)
|
||||||
|
|
||||||
imported = 0
|
imported = 0
|
||||||
conflicts = 0
|
conflicts = 0
|
||||||
|
timeouts = 0
|
||||||
|
errors = 0
|
||||||
|
|
||||||
for component in cal.walk():
|
for component in cal.walk():
|
||||||
if component.name != "VEVENT":
|
if component.name != "VEVENT":
|
||||||
@@ -93,25 +167,30 @@ def import_ics_to_caldav(ics_bytes: bytes, uid_hash: str):
|
|||||||
single_cal.add("version", "2.0")
|
single_cal.add("version", "2.0")
|
||||||
single_cal.add_component(component)
|
single_cal.add_component(component)
|
||||||
|
|
||||||
uid = str(component.get("uid", uid_hash + f"-{imported + conflicts}"))
|
uid = str(component.get("uid", uid_hash + f"-{imported + conflicts + timeouts + errors}"))
|
||||||
summary = component.get("summary", "(kein Titel)")
|
summary = str(component.get("summary", "(kein Titel)"))
|
||||||
sequence = component.get("sequence", "(keine sequence)")
|
sequence = component.get("sequence", "(keine sequence)")
|
||||||
|
result = add_event_with_retries(
|
||||||
|
calendar,
|
||||||
|
single_cal.to_ical().decode("utf-8"),
|
||||||
|
summary,
|
||||||
|
uid,
|
||||||
|
sequence,
|
||||||
|
timeout,
|
||||||
|
retries,
|
||||||
|
backoff,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
if result == "imported":
|
||||||
calendar.add_event(single_cal.to_ical().decode("utf-8"))
|
|
||||||
log.info(f" ✓ Termin importiert: {summary} [{uid[:20]}]")
|
|
||||||
imported += 1
|
imported += 1
|
||||||
except Exception as e:
|
elif result == "conflict":
|
||||||
if is_newer_version_conflict(e):
|
conflicts += 1
|
||||||
log.info(
|
elif result == "timeout":
|
||||||
f" ↷ Konflikt ignoriert: Neuere Version existiert bereits: {summary} "
|
timeouts += 1
|
||||||
f"[{uid[:20]}], SEQUENCE={sequence}"
|
else:
|
||||||
)
|
errors += 1
|
||||||
conflicts += 1
|
|
||||||
else:
|
|
||||||
log.warning(f" ✗ Fehler beim Import von {uid[:20]}: {e}")
|
|
||||||
|
|
||||||
return imported, conflicts
|
return imported, conflicts, timeouts, errors
|
||||||
|
|
||||||
|
|
||||||
def process_mailbox():
|
def process_mailbox():
|
||||||
@@ -143,6 +222,8 @@ def process_mailbox():
|
|||||||
imported_hashes = load_seen_uids()
|
imported_hashes = load_seen_uids()
|
||||||
total_imported = 0
|
total_imported = 0
|
||||||
total_conflicts = 0
|
total_conflicts = 0
|
||||||
|
total_timeouts = 0
|
||||||
|
total_errors = 0
|
||||||
processed_mail_count = 0
|
processed_mail_count = 0
|
||||||
|
|
||||||
for msg_uid in message_uids:
|
for msg_uid in message_uids:
|
||||||
@@ -181,9 +262,11 @@ def process_mailbox():
|
|||||||
log.info(f" ICS-Anhang gefunden: {filename!r} ({uid_hash[:12]}…)")
|
log.info(f" ICS-Anhang gefunden: {filename!r} ({uid_hash[:12]}…)")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
imported_count, conflict_count = import_ics_to_caldav(ics_bytes, uid_hash)
|
imported_count, conflict_count, timeout_count, error_count = import_ics_to_caldav(ics_bytes, uid_hash)
|
||||||
total_imported += imported_count
|
total_imported += imported_count
|
||||||
total_conflicts += conflict_count
|
total_conflicts += conflict_count
|
||||||
|
total_timeouts += timeout_count
|
||||||
|
total_errors += error_count
|
||||||
|
|
||||||
if imported_count > 0 or conflict_count > 0:
|
if imported_count > 0 or conflict_count > 0:
|
||||||
save_uid(uid_hash)
|
save_uid(uid_hash)
|
||||||
@@ -191,7 +274,7 @@ def process_mailbox():
|
|||||||
mail_processed = True
|
mail_processed = True
|
||||||
else:
|
else:
|
||||||
log.warning(
|
log.warning(
|
||||||
f" Kein VEVENT importiert oder als Konflikt erkannt für Anhang {filename!r}; Mail bleibt ungelesen."
|
f" Kein VEVENT erfolgreich verarbeitet für Anhang {filename!r}; Mail bleibt ungelesen."
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -215,6 +298,8 @@ def process_mailbox():
|
|||||||
|
|
||||||
log.info(f"Fertig. {total_imported} Termin(e) importiert.")
|
log.info(f"Fertig. {total_imported} Termin(e) importiert.")
|
||||||
log.info(f"{total_conflicts} Konflikt(e) als bereits aktueller Serverstand erkannt.")
|
log.info(f"{total_conflicts} Konflikt(e) als bereits aktueller Serverstand erkannt.")
|
||||||
|
log.info(f"{total_timeouts} Timeout-Fall/Fälle beim CalDAV-Upload.")
|
||||||
|
log.info(f"{total_errors} sonstige Fehler beim CalDAV-Upload.")
|
||||||
log.info(f"{processed_mail_count} Mail(s) als gelesen markiert.")
|
log.info(f"{processed_mail_count} Mail(s) als gelesen markiert.")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user