From 01665b6e3ab2e627f866596d5340891d014bb8a6 Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Thu, 16 Oct 2025 22:29:04 +0200 Subject: [PATCH] Fixes POST request 403 errors on instances behind Cloudflare proxy (#368) * Add full headers to ingestor POST requests to avoid CF bans * run black * Guard Authorization header when token absent --------- Co-authored-by: varna9000 --- data/mesh_ingestor/queue.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/data/mesh_ingestor/queue.py b/data/mesh_ingestor/queue.py index fb63acc..b8e323e 100644 --- a/data/mesh_ingestor/queue.py +++ b/data/mesh_ingestor/queue.py @@ -72,11 +72,25 @@ def _post_json( return url = f"{instance}{path}" data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - url, data=data, headers={"Content-Type": "application/json"} - ) + + # Add full headers to avoid Cloudflare blocks on instances behind cloudflare proxy + headers = { + "Content-Type": "application/json", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "application/json", + "Accept-Language": "en-US,en;q=0.9", + "Origin": f"{instance}", + "Referer": f"{instance}", + } if api_token: - req.add_header("Authorization", f"Bearer {api_token}") + headers["Authorization"] = f"Bearer {api_token}" + + req = urllib.request.Request( + url, + data=data, + headers=headers, + ) + try: with urllib.request.urlopen(req, timeout=10) as resp: resp.read()