Runging own Telgram bot based on PTB 13x and local Telegram bot API behind nginx

Question:

Try to use local Telegram bot API to take benefits of larger files for my bot serving and helping users in supergroup with files as described here
Build stack with Telegram Bot API, nginx as reverse proxy and my bot works ok, based on docker-compose template:

version: '3.7'

networks:
  int:
    name: internal
    driver: bridge

services:
  api:
    hostname: local-api
    image: aiogram/telegram-bot-api:latest
    restart: always
    networks:
      - int
    environment:
      TELEGRAM_API_ID: BOT_ID
      TELEGRAM_API_HASH: BOT_HASH
      TELEGRAM_LOCAL: "true"
      TELEGRAM_HTTP_IP_ADDRESS: 0.0.0.0
      TELEGRAM_HTTP_PORT: 8081
      TELEGRAM_MAX_FILESIZE: 1000000000 # задава максимален размер на файла на 1GB
    volumes:
      - telegram-bot-api-data:/var/lib/telegram-bot-api

  nginx:
    hostname: nginx
    image: nginx:1.19-alpine
    restart: always
    depends_on:
      - api
    volumes:
      - telegram-bot-api-data:/var/lib/telegram-bot-api
      - ./nginx:/etc/nginx/conf.d/
    ports:
      - "88:88"
    networks:
      - int

  telegram-bot:
    depends_on:
     - api
     - nginx
    hostname : telegram-bot
    networks:
      - int
    restart: unless-stopped
    build:
      context: /root/docker/telegram-bot
      dockerfile: Dockerfile
    image: telegram-bot/latest
    environment:
      LOCAL_API_URL: http://nginx:88/bot
      LOCAL_API_FILES: http://nginx:88/file/bot
    labels:
      - "com.centurylinklabs.watchtower.enable=false"
    volumes:
      - /root/docker/telegram-bot/:/app/

volumes:
  telegram-bot-api-data:

And accoring nginx config with redirects:

# use $sanitized_request instead of $request to hide Telegram token
log_format token_filter '$remote_addr - $remote_user [$time_local] '
                        '"$sanitized_request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent"';

upstream telegram-bot-api {
    server local-api:8081;
}

server {
    listen 88;

    chunked_transfer_encoding on;
    proxy_connect_timeout 600;
    proxy_send_timeout 600;
    proxy_read_timeout 600;
    send_timeout 600;
    client_max_body_size 2G;
    client_body_buffer_size 30M;
    keepalive_timeout 0;
    proxy_http_version 1.1;

    rewrite_log on;

    set $sanitized_request $request;
    if ( $sanitized_request ~ (w+)s(/botd+):[-w]+/(S+)s(.*) ) {
        set $sanitized_request "$1 $2:<hidden-token>/$3 $4";
    }
    access_log /var/log/nginx/access.log token_filter;
    error_log /var/log/nginx/error.log notice;

    location ~* /file/botd+:(.*) {
        rewrite ^/file/bot(.*) /$1 break;
        proxy_http_version  1.1;
        try_files $uri @files;
    }

    location / {
        try_files $uri @api;
        proxy_http_version  1.1;
    }

    location @files {
        root /var/lib/telegram-bot-api;
        gzip on;
        gzip_vary on;
        gzip_proxied any;
        gzip_comp_level 6;
        gzip_buffers 64 8k;
        gzip_http_version 1.1;
        gzip_min_length 1100;
    }

    location @api {
        proxy_pass  http://telegram-bot-api;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
    }
}

At last Telegram bot is setup to work with local Telegram Bot API:

.env
LOCAL_API=True
LOCAL_API_URL=http://nginx:88/bot
LOCAL_API_FILES=http://nginx:88/file/bot

PTB telegram bot setup

updater = Updater(config["BOT_TOKEN"], use_context = True, base_url = config["LOCAL_API_URL"], base_file_url = config["LOCAL_API_FILES"], arbitrary_callback_data = True)

Things works fine but file redirect wont work as expect – the limit is still on and can’t get local link for received from bot file

2023/04/04 11:15:52 [error] 30#30: *15 open()
"/var/lib/telegram-bot-api/TOKEN_ID:TOKEN_HASH/var/lib/telegram-bot-api/TOKEN_ID:TOKEN_HASH/documents/file_16.apk"
failed (2: No such file or directory), client: 172.19.0.4, server: _,
request: "GET
/file/botTOKEN_ID%3ATOKEN_HASH//var/lib/telegram-bot-api/TOKEN_ID%3ATOKEN_HASH/documents/file_16.apk
HTTP/1.1", host: "nginx:81"
172.19.0.4 – – [04/Apr/2023:11:15:52 +0000] "GET /file/botTOKEN_ID%3ATOKEN_HASH//var/lib/telegram-bot-api/TOKEN_ID%3ATOKEN_HASH/documents/file_16.apk
HTTP/1.1" 404 154 "-" "Python Telegram Bot
(https://github.com/python-telegram-bot/python-telegram-bot)"

There is "dublicate path" in logs
/var/lib/telegram-bot-api/TOKEN_ID:TOKEN_HASH/var/lib/telegram-bot-api/TOKEN_ID:TOKEN_HASH/documents/file_16.apk

But despite of that curl request works as expected

curl -i http://nginx:81/file/botTOKEN_ID:TOKEN_HASH/documents/file_16.apk

HTTP/1.1 200 OK Server: nginx/1.19.10 Date: Tue, 04 Apr 2023 14:31:01
GMT Content-Type: application/octet-stream Content-Length: 10066456
Last-Modified: Tue, 04 Apr 2023 06:26:47 GMT Connection: close ETag:
"642bc327-999a18" Accept-Ranges: bytes

Warning: Binary output can mess up your terminal. Use "–output -" to
tell Warning: curl to output it to your terminal anyway, or consider
"–output Warning: " to save to a file.

Can’t realize from where become error, bot itslef request to local API or nginx redirects ?
Rewriting bot at PTB 20.x is not an option do not have enough time for probono project.
Any ideas are welcome!

Edit: Wrong path comes form context.bot.get_file(file_id) and return file_path !

Asked By: Topper

||

Answers:

To others who faced that usecase:

nginx.conf

# use $sanitized_request instead of $request to hide Telegram token
log_format token_filter '$remote_addr - $remote_user [$time_local] '
                        '"$sanitized_request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent"';

upstream telegram-bot-api {
    server local-api:8081;
}

server {
    listen 88;
    listen 8443 ssl;

    ssl_certificate /etc/nginx/conf.d/PUBLIC.pem;
    ssl_certificate_key /etc/nginx/conf.d/PRIVATE.key;
    ssl_session_timeout 5m;
    ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
    ssl_prefer_server_ciphers on;

    server_name api.thetopper.eu;

    chunked_transfer_encoding on;
    proxy_connect_timeout 600;
    proxy_send_timeout 600;
    proxy_read_timeout 600;
    send_timeout 600;
    client_max_body_size 2G;
    client_body_buffer_size 30M;
    keepalive_timeout 0;

    set $sanitized_request $request;
    if ( $sanitized_request ~ (w+)s(/botd+):[-w]+/(S+)s(.*) ) {
        set $sanitized_request "$1 $2:<hidden-token>/$3 $4";
    }
    access_log /var/log/nginx/access.log token_filter;

    location ~* /file/botd+:(.*) {
        rewrite ^/file/bot(.*) /$1 break;
        try_files $uri @files;
    }

    location / {
        try_files $uri @api;
    }

    location @files {
        root /var/lib/telegram-bot-api;
        gzip on;
        gzip_vary on;
        gzip_proxied any;
        gzip_comp_level 6;
        gzip_buffers 64 8k;
        gzip_http_version 1.1;
        gzip_min_length 1100;
        # Remove /var/lib/telegram-bot-api/TOKEN_ID:TOKEN_HASH prefix from file path
        #rewrite ^/file/bot(.*)/(.*)$ /$1/$2 break;
        # Log $uri variable to Nginx error log
        error_log /var/log/nginx/error.log info;
        error_log /var/log/nginx/error.log debug_http;
    }

    location @api {
        proxy_pass  http://telegram-bot-api;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
    }
}

config.env

LOCAL_API=True
LOCAL_API_URL=http://nginx:88/bot
LOCAL_API_FILES=http://nginx:88/file/bot

bot.py (PTB v13.15)

updater = Updater(config["BOT_TOKEN"], use_context=True, base_url=config["LOCAL_API_URL"],arbitrary_callback_data=True, base_file_url=config["LOCAL_API_FILES"])
....
file_name = update.message.document.file_name
file_name_id = update.message.document.file_id
file_path = f"/var/lib/telegram-bot-api/{config["BOT_TOKEN"]}/documents/{update.message.document.file_id.file_path.split('/')[-1]}"
Answered By: Topper