feat: initial versions of core components

src/optipng.py

@@ -0,0 +1,72 @@
+import subprocess
+import os
+import logging
+
+import magic
+
+
+_logger = logging.getLogger(__name__)
+_logger.addHandler(logging.NullHandler())
+
+# TODO: потребляет очень мало памяти,
+# можно использовать много потоков без проблем
+_CPU_COUNT = len(os.sched_getaffinity(0))
+
+
+def _check_preconditions():
+    assert subprocess.check_call(["optipng", "--version"]) == 0
+
+
+def _walk_for_png(base_path: str):
+    if not os.path.isdir(base_path):
+        raise ValueError(f"Base path: {base_path} is not a dir")
+    for root, _, files in os.walk(base_path):
+        for filename in files:
+            if not filename.lower().endswith("png"):
+                continue
+            filepath = os.path.join(root, filename)
+            filetype = magic.from_file(filepath, mime=True)
+            if not filetype.endswith("/png"):
+                _logger.warning(f".png extension but not png type: {filetype} for file: {filepath}")
+                continue
+            yield filepath
+
+
+def apply_optipng(path: str, level: int) -> (int, int):
+    assert 0 <= level <= 7
+    intital_size = os.path.getsize(path)
+    if intital_size == 0:
+        _logger.warning(f"Empty PNG at {path}")
+    subprocess.check_call(
+        ["optipng", f"-o{level}", path],
+        stderr=subprocess.DEVNULL,  # optipng writes log to stderr
+    )
+    final_size = os.path.getsize(path)
+    _logger.debug(f"optipng -o{level} result for {path} is {1. * final_size / intital_size}")
+    return intital_size, final_size
+
+
+def apply_optipng_recursive(
+    base_path: str, level: int = 7, threads: int = _CPU_COUNT
+) -> (int, int):
+    intital_size, final_size = 0, 0
+    for path in _walk_for_png(base_path):
+        try:
+            cur_initial_size, cur_final_size = apply_optipng(path, level=level)
+        except subprocess.CalledProcessError:
+            _logger.error(f"Unable to process file: {path}")
+            continue
+        intital_size += cur_initial_size
+        final_size += cur_final_size
+
+    return intital_size, final_size
+
+
+_check_preconditions()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    print(len(list(_walk_for_png("../../Projects/FiraCode/"))))
+    print(list(_walk_for_png(".")))
+    print(apply_optipng_recursive("test_dir_base_ready/"))

src/rabbit.py

@@ -0,0 +1,88 @@
+import logging
+import datetime
+from typing import Optional
+
+import pika
+from pika import BlockingConnection
+from pika.channel import Channel
+from pika.credentials import PlainCredentials
+from pika.spec import BasicProperties
+
+from src.settings import (
+    RABBIT_HOST,
+    RABBIT_PORT,
+    RABBIT_CREDENTIALS,
+    RABBIT_TASK_QUEUE,
+    RABBIT_REPLY_QUEUE,
+    MESSAGE_TTL,
+)
+from src.messages import AnalyzeTask, AnalyzeResponse
+
+_logger = logging.getLogger(__name__)
+_logger.addHandler(logging.NullHandler())
+
+CONNECTION_ATTEMPTS = 3
+QUEUE_MAX_PRIORITY = 4
+RABBIT_MESSAGE_TTL = str(int(MESSAGE_TTL * 1000))
+
+
+def get_connection() -> BlockingConnection:
+    _logger.info(f"connecting to RabbitMQ at {RABBIT_HOST}:{RABBIT_PORT}")
+    connection = pika.BlockingConnection(
+        pika.ConnectionParameters(
+            host=RABBIT_HOST,
+            port=RABBIT_PORT,
+            credentials=PlainCredentials(*RABBIT_CREDENTIALS),
+            connection_attempts=CONNECTION_ATTEMPTS,
+        )
+    )
+    return connection
+
+
+def get_channel(connection: Optional[BlockingConnection] = None) -> Channel:
+    if connection:
+        _connection = connection
+    else:
+        _connection = get_connection()
+
+    channel = _connection.channel()
+    base_queue_params = dict(
+        durable=True,
+    )
+
+    if QUEUE_MAX_PRIORITY > 1:
+        base_queue_params["arguments"] = {"x-max-priority": QUEUE_MAX_PRIORITY}
+    channel.queue_declare(queue=RABBIT_REPLY_QUEUE, **base_queue_params)
+    channel.queue_declare(queue=RABBIT_TASK_QUEUE, **base_queue_params)
+
+    channel.basic_qos(prefetch_count=1)
+    return channel
+
+
+def send_task(channel: Channel, data: bytes):
+    channel.basic_publish(
+        exchange="",
+        routing_key=RABBIT_TASK_QUEUE,
+        body=data,
+        properties=BasicProperties(expiration=RABBIT_MESSAGE_TTL, reply_to=RABBIT_REPLY_QUEUE),
+    )
+
+
+def send_reply(channel, data: bytes):
+    channel.basic_publish(
+        exchange="",
+        routing_key=RABBIT_REPLY_QUEUE,
+        body=data,
+        properties=BasicProperties(expiration=RABBIT_MESSAGE_TTL),
+    )
+
+
+def consume_task(channel, queue: str, timeout=None, auto_ack=True, max_count=None):
+    for method, properties, body in channel.consume(
+        queue, auto_ack=auto_ack, inactivity_timeout=timeout
+    ):
+        yield body
+        if max_count and method.delivery_tag == max_count:
+            break
+    requeued_messages = channel.cancel()
+    _logger.info(f"Requeued {requeued_messages} messages")

src/settings.py

@@ -0,0 +1,57 @@
+import logging
+import datetime
+import os
+import sys
+import platform
+
+
+DEBUG = bool(os.getenv("DEBUG"))
+WORKER_NAME = os.getenv("WORKER_NAME", platform.node())
+
+DB_PATH = os.getenv("DB_PATH", "sqlite:///assets_bot.db")
+
+MESSAGE_TTL = datetime.timedelta(hours=12).total_seconds()
+
+PREPROCESS_LEVEL = 2
+
+THRESHOLD_ABSOLUTE = 50 * 1024  # 50 KiB
+THRESHOLD_RELATIVE = 0.1  # 10% or better
+
+GIT_PRIVATE_KEY_PATH = os.getenv("GIT_PRIVATE_KEY_PATH", "z_assets-optimizer")
+
+PR_TITLE = "PNG optimization"
+PR_BODY_TEMPLATE = """
+Hello.
+
+This automated tool has found possible size optimization for {relative:.2f}% or {absolute} total.
+It is just calls of `optipng -o7`, lossless optimization
+
+If you want to opt out of PRs like this or want to provide feedback, please contact @AlekseyLobanov.
+""".strip()
+
+RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1")
+RABBIT_PORT = int(os.getenv("RABBIT_PORT", 5672))
+RABBIT_CREDENTIALS = ("alex", "testtest")
+_RABBIT_BASE_QUEUE = "tasks.analyze"
+RABBIT_TASK_QUEUE = _RABBIT_BASE_QUEUE + ".in"
+RABBIT_REPLY_QUEUE = _RABBIT_BASE_QUEUE + ".out"
+
+LOGGING_FORMAT = "%(asctime)s.%(msecs)03d %(levelname)s %(module)s: %(message)s"
+LOGGING_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+LOGS_PATH = '/var/log/assets-bot'
+
+
+def init_logging():
+    if DEBUG:
+        logging.basicConfig(format=LOGGING_FORMAT, datefmt=LOGGING_DATE_FORMAT, level=logging.INFO)
+    else:
+        out_path = os.path.join(
+            LOGS_PATH,
+            os.path.basename(os.path.normpath(sys.argv[0].replace(".py", ".log")))
+        )
+        logging.basicConfig(
+            filename=out_path,
+            format=LOGGING_FORMAT,
+            datefmt=LOGGING_DATE_FORMAT,
+            level=logging.INFO,
+        )