From ca37d31e184b194a3005bfcd5629f6e9afd34e9c Mon Sep 17 00:00:00 2001 From: Aleksey Lobanov Date: Sun, 3 Jan 2021 15:59:33 +0300 Subject: [PATCH] feat: initial versions of core components --- src/optipng.py | 72 ++++++++++++++++++++++++++++++++++++++++ src/rabbit.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ src/settings.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 src/optipng.py create mode 100644 src/rabbit.py create mode 100644 src/settings.py diff --git a/src/optipng.py b/src/optipng.py new file mode 100644 index 0000000..e2bd915 --- /dev/null +++ b/src/optipng.py @@ -0,0 +1,72 @@ +import subprocess +import os +import logging + +import magic + + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + +# TODO: потребляет очень мало памяти, +# можно использовать много потоков без проблем +_CPU_COUNT = len(os.sched_getaffinity(0)) + + +def _check_preconditions(): + assert subprocess.check_call(["optipng", "--version"]) == 0 + + +def _walk_for_png(base_path: str): + if not os.path.isdir(base_path): + raise ValueError(f"Base path: {base_path} is not a dir") + for root, _, files in os.walk(base_path): + for filename in files: + if not filename.lower().endswith("png"): + continue + filepath = os.path.join(root, filename) + filetype = magic.from_file(filepath, mime=True) + if not filetype.endswith("/png"): + _logger.warning(f".png extension but not png type: {filetype} for file: {filepath}") + continue + yield filepath + + +def apply_optipng(path: str, level: int) -> (int, int): + assert 0 <= level <= 7 + intital_size = os.path.getsize(path) + if intital_size == 0: + _logger.warning(f"Empty PNG at {path}") + subprocess.check_call( + ["optipng", f"-o{level}", path], + stderr=subprocess.DEVNULL, # optipng writes log to stderr + ) + final_size = os.path.getsize(path) + _logger.debug(f"optipng -o{level} result for {path} is {1. * final_size / intital_size}") + return intital_size, final_size + + +def apply_optipng_recursive( + base_path: str, level: int = 7, threads: int = _CPU_COUNT +) -> (int, int): + intital_size, final_size = 0, 0 + for path in _walk_for_png(base_path): + try: + cur_initial_size, cur_final_size = apply_optipng(path, level=level) + except subprocess.CalledProcessError: + _logger.error(f"Unable to process file: {path}") + continue + intital_size += cur_initial_size + final_size += cur_final_size + + return intital_size, final_size + + +_check_preconditions() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + print(len(list(_walk_for_png("../../Projects/FiraCode/")))) + print(list(_walk_for_png("."))) + print(apply_optipng_recursive("test_dir_base_ready/")) diff --git a/src/rabbit.py b/src/rabbit.py new file mode 100644 index 0000000..b3d8e68 --- /dev/null +++ b/src/rabbit.py @@ -0,0 +1,88 @@ +import logging +import datetime +from typing import Optional + +import pika +from pika import BlockingConnection +from pika.channel import Channel +from pika.credentials import PlainCredentials +from pika.spec import BasicProperties + +from src.settings import ( + RABBIT_HOST, + RABBIT_PORT, + RABBIT_CREDENTIALS, + RABBIT_TASK_QUEUE, + RABBIT_REPLY_QUEUE, + MESSAGE_TTL, +) +from src.messages import AnalyzeTask, AnalyzeResponse + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + +CONNECTION_ATTEMPTS = 3 +QUEUE_MAX_PRIORITY = 4 +RABBIT_MESSAGE_TTL = str(int(MESSAGE_TTL * 1000)) + + +def get_connection() -> BlockingConnection: + _logger.info(f"connecting to RabbitMQ at {RABBIT_HOST}:{RABBIT_PORT}") + connection = pika.BlockingConnection( + pika.ConnectionParameters( + host=RABBIT_HOST, + port=RABBIT_PORT, + credentials=PlainCredentials(*RABBIT_CREDENTIALS), + connection_attempts=CONNECTION_ATTEMPTS, + ) + ) + return connection + + +def get_channel(connection: Optional[BlockingConnection] = None) -> Channel: + if connection: + _connection = connection + else: + _connection = get_connection() + + channel = _connection.channel() + base_queue_params = dict( + durable=True, + ) + + if QUEUE_MAX_PRIORITY > 1: + base_queue_params["arguments"] = {"x-max-priority": QUEUE_MAX_PRIORITY} + channel.queue_declare(queue=RABBIT_REPLY_QUEUE, **base_queue_params) + channel.queue_declare(queue=RABBIT_TASK_QUEUE, **base_queue_params) + + channel.basic_qos(prefetch_count=1) + return channel + + +def send_task(channel: Channel, data: bytes): + channel.basic_publish( + exchange="", + routing_key=RABBIT_TASK_QUEUE, + body=data, + properties=BasicProperties(expiration=RABBIT_MESSAGE_TTL, reply_to=RABBIT_REPLY_QUEUE), + ) + + +def send_reply(channel, data: bytes): + channel.basic_publish( + exchange="", + routing_key=RABBIT_REPLY_QUEUE, + body=data, + properties=BasicProperties(expiration=RABBIT_MESSAGE_TTL), + ) + + +def consume_task(channel, queue: str, timeout=None, auto_ack=True, max_count=None): + for method, properties, body in channel.consume( + queue, auto_ack=auto_ack, inactivity_timeout=timeout + ): + yield body + if max_count and method.delivery_tag == max_count: + break + requeued_messages = channel.cancel() + _logger.info(f"Requeued {requeued_messages} messages") diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 0000000..2f09379 --- /dev/null +++ b/src/settings.py @@ -0,0 +1,57 @@ +import logging +import datetime +import os +import sys +import platform + + +DEBUG = bool(os.getenv("DEBUG")) +WORKER_NAME = os.getenv("WORKER_NAME", platform.node()) + +DB_PATH = os.getenv("DB_PATH", "sqlite:///assets_bot.db") + +MESSAGE_TTL = datetime.timedelta(hours=12).total_seconds() + +PREPROCESS_LEVEL = 2 + +THRESHOLD_ABSOLUTE = 50 * 1024 # 50 KiB +THRESHOLD_RELATIVE = 0.1 # 10% or better + +GIT_PRIVATE_KEY_PATH = os.getenv("GIT_PRIVATE_KEY_PATH", "z_assets-optimizer") + +PR_TITLE = "PNG optimization" +PR_BODY_TEMPLATE = """ +Hello. + +This automated tool has found possible size optimization for {relative:.2f}% or {absolute} total. +It is just calls of `optipng -o7`, lossless optimization + +If you want to opt out of PRs like this or want to provide feedback, please contact @AlekseyLobanov. +""".strip() + +RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1") +RABBIT_PORT = int(os.getenv("RABBIT_PORT", 5672)) +RABBIT_CREDENTIALS = ("alex", "testtest") +_RABBIT_BASE_QUEUE = "tasks.analyze" +RABBIT_TASK_QUEUE = _RABBIT_BASE_QUEUE + ".in" +RABBIT_REPLY_QUEUE = _RABBIT_BASE_QUEUE + ".out" + +LOGGING_FORMAT = "%(asctime)s.%(msecs)03d %(levelname)s %(module)s: %(message)s" +LOGGING_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +LOGS_PATH = '/var/log/assets-bot' + + +def init_logging(): + if DEBUG: + logging.basicConfig(format=LOGGING_FORMAT, datefmt=LOGGING_DATE_FORMAT, level=logging.INFO) + else: + out_path = os.path.join( + LOGS_PATH, + os.path.basename(os.path.normpath(sys.argv[0].replace(".py", ".log"))) + ) + logging.basicConfig( + filename=out_path, + format=LOGGING_FORMAT, + datefmt=LOGGING_DATE_FORMAT, + level=logging.INFO, + )