Update plugins, refactor proxy plugin, yt preview now gets lyrics

2024-03-04 00:53:58 -07:00
parent e8186c9fec
commit d2acef611b
8 changed files with 227 additions and 74 deletions
--- a/plugins/proxy.py
+++ b/plugins/proxy.py
@@ -9,44 +9,50 @@ import logging
 import random
 import requests
 import socket
-import ssl
 import time
 from datetime import datetime, timedelta
+import concurrent.futures
 import simplematrixbotlib as botlib
+import sqlite3

 # Constants
 SOCKS5_LIST_URL = 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt'
-MAX_TRIES = 5
+# SOCKS5_LIST_URL = 'https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/protocols/socks5/data.txt'
+MAX_TRIES = 64
 PROXY_LIST_FILENAME = 'socks5.txt'
 PROXY_LIST_EXPIRATION = timedelta(hours=8)
+MAX_THREADS = 128
+PROXIES_DB_FILE = 'proxies.db'
+MAX_PROXIES_IN_DB = 500

-async def test_socks5_proxy(proxy):
+# Setup verbose logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def test_proxy(proxy):
    """
-    Test a SOCKS5 proxy by attempting a connection and sending a request through it.
+    Test a SOCKS5 proxy and return the outcome.

    Args:
        proxy (str): The SOCKS5 proxy address in the format 'ip:port'.

    Returns:
-        bool: True if the proxy is working, False otherwise.
-        float: The latency in milliseconds if the proxy is working, None otherwise.
+        tuple: (bool: success, str: proxy, int: latency)
    """
    try:
        ip, port = proxy.split(':')
+        logging.info(f"Testing SOCKS5 proxy: {ip}:{port}")
        start_time = time.time()
-        with socket.create_connection((ip, int(port)), timeout=5) as client:
+        with socket.create_connection((ip, int(port)), timeout=12) as client:
            client.sendall(b'\x05\x01\x00')
            response = client.recv(2)
            if response == b'\x05\x00':
                latency = int(round((time.time() - start_time) * 1000, 0))
-                logging.info(f"Successful connection to SOCKS5 proxy {proxy}. Latency: {latency} ms")
-                return True, latency
+                return True, proxy, latency
            else:
-                logging.info(f"Failed to connect to SOCKS5 proxy {proxy}: Connection refused")
-                return False, None
+                return False, proxy, None
    except Exception as e:
-        logging.error(f"Error testing SOCKS5 proxy {proxy}: {e}")
-        return False, None
+        return False, proxy, None
+

 async def download_proxy_list():
    """
@@ -72,6 +78,69 @@ async def download_proxy_list():
        return False


+def check_db_for_proxy():
+    """
+    Check the proxies database for a working proxy.
+    If found, test the proxy and remove it from the database if it doesn't work.
+
+    Returns:
+        str or None: The working proxy if found, None otherwise.
+    """
+    try:
+        with sqlite3.connect(PROXIES_DB_FILE) as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                CREATE TABLE IF NOT EXISTS proxies (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    proxy TEXT,
+                    latency INTEGER,
+                    status TEXT
+                )
+            """)
+            cursor.execute("SELECT proxy, latency FROM proxies WHERE status = 'working' AND latency < 3000 ORDER BY RANDOM() LIMIT 1")
+            result = cursor.fetchone()
+            if result:
+                proxy, latency = result
+                success, _, _ = test_proxy(proxy)
+                if success:
+                    return proxy, latency
+                else:
+                    cursor.execute("DELETE FROM proxies WHERE proxy = ?", (proxy,))
+                    conn.commit()
+                    logging.info(f"Removed non-working proxy from the database: {proxy}")
+                    return None, None
+            else:
+                return None, None
+    except Exception as e:
+        logging.error(f"Error checking proxies database: {e}")
+        return None, None
+
+
+def save_proxy_to_db(proxy, latency):
+    """
+    Save a working proxy to the proxies database.
+
+    Args:
+        proxy (str): The working proxy to be saved.
+        latency (int): Latency of the proxy.
+    """
+    try:
+        with sqlite3.connect(PROXIES_DB_FILE) as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                CREATE TABLE IF NOT EXISTS proxies (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    proxy TEXT,
+                    latency INTEGER,
+                    status TEXT
+                )
+            """)
+            cursor.execute("INSERT INTO proxies (proxy, latency, status) VALUES (?, ?, 'working')", (proxy, latency))
+            conn.commit()
+    except Exception as e:
+        logging.error(f"Error saving proxy to database: {e}")
+
+
 async def handle_command(room, message, bot, prefix, config):
    """
    Function to handle the !proxy command.
@@ -87,34 +156,58 @@ async def handle_command(room, message, bot, prefix, config):
    if match.is_not_from_this_bot() and match.prefix() and match.command("proxy"):
        logging.info("Received !proxy command")

-        # Download proxy list if needed
-        if not await download_proxy_list():
-            await bot.api.send_markdown_message(room.room_id, "Error downloading proxy list")
-            logging.error("Error downloading proxy list")
+        # Check database for a working proxy
+        working_proxy, latency = check_db_for_proxy()
+        if working_proxy:
+            await bot.api.send_markdown_message(room.room_id,
+                                                 f"✅ Using cached working SOCKS5 Proxy: **{working_proxy}** - Latency: **{latency} ms**")
+            logging.info(f"Using cached working SOCKS5 proxy {working_proxy}")
            return

-        try:
-            # Read proxies from file
-            with open(PROXY_LIST_FILENAME, 'r') as f:
-                socks5_proxies = f.read().splitlines()
-            random.shuffle(socks5_proxies)
+        # Download proxy list if needed
+        else:
+            if not await download_proxy_list():
+                await bot.api.send_markdown_message(room.room_id, "Error downloading proxy list")
+                logging.error("Error downloading proxy list")
+                return

-            # Test proxies
-            socks5_proxy = None
-            for proxy in socks5_proxies[:MAX_TRIES]:
-                success, latency = await test_socks5_proxy(proxy)
-                if success:
-                    socks5_proxy = proxy
-                    break
+            try:
+                # Read proxies from file
+                with open(PROXY_LIST_FILENAME, 'r') as f:
+                    socks5_proxies = [line.replace("socks5://", "") for line in f.read().splitlines()]
+                random.shuffle(socks5_proxies)
+
+                # Test proxies concurrently
+                tested_proxies = 0
+                with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
+                    futures = []
+                    for proxy in socks5_proxies[:MAX_TRIES]:
+                        futures.append(executor.submit(test_proxy, proxy))
+                    for future in concurrent.futures.as_completed(futures):
+                        success, proxy, latency = future.result()
+                        if success:
+                            await bot.api.send_markdown_message(room.room_id,
+                                                                f"✅ Anonymous SOCKS5 Proxy: **{proxy}** - Latency: **{latency} ms**")
+                            logging.info(f"Sent SOCKS5 proxy {proxy} to the room")
+                            save_proxy_to_db(proxy, latency)  # Save working proxy to the database
+                            tested_proxies += 1
+                            if tested_proxies >= MAX_PROXIES_IN_DB:
+                                break  # Stop testing proxies once MAX_PROXIES_IN_DB are saved to the database
+
+                # Check database for a working proxy after testing
+                working_proxy, latency = check_db_for_proxy()
+                if working_proxy:
+                    await bot.api.send_markdown_message(room.room_id,
+                                                        f"✅ Using cached working SOCKS5 Proxy: **{working_proxy}** - Latency: **{latency} ms**")
+                    logging.info(f"Using cached working SOCKS5 proxy {working_proxy}")
+
+                else:
+                    # If no working proxy found after testing
+                    await bot.api.send_markdown_message(room.room_id, "❌ No working anonymous SOCKS5 proxy found")
+                    logging.info("No working anonymous SOCKS5 proxy found")
+
+            except Exception as e:
+                logging.error(f"Error handling !proxy command: {e}")
+                await bot.api.send_markdown_message(room.room_id, "❌ Error handling !proxy command")

-            # Send the first working anonymous proxy of each type to the Matrix room
-            if socks5_proxy:
-                await bot.api.send_markdown_message(room.room_id, f"✅ Anonymous SOCKS5 Proxy: **{socks5_proxy}** - Latency: **{latency} ms**")
-                logging.info(f"Sent SOCKS5 proxy {socks5_proxy} to the room")
-            else:
-                await bot.api.send_markdown_message(room.room_id, "❌ No working anonymous SOCKS5 proxy found")
-                logging.info("No working anonymous SOCKS5 proxy found")

-        except Exception as e:
-            logging.error(f"Error handling !proxy command: {e}")
-            await bot.api.send_markdown_message(room.room_id, "❌ Error handling !proxy command")