Forráskód Böngészése

Add better logging and some env configuration

Colin Powell 3 éve
szülő
commit
8740fe8284
5 módosított fájl, 156 hozzáadás és 16 törlés
  1. 1 0
      .gitignore
  2. 5 0
      emus.conf.example
  3. 89 15
      emus/settings.py
  4. 58 1
      poetry.lock
  5. 3 0
      pyproject.toml

+ 1 - 0
.gitignore

@@ -1,2 +1,3 @@
 db.sqlite3
+emus.conf
 media/roms/virtualboy

+ 5 - 0
emus.conf.example

@@ -0,0 +1,5 @@
+# You can use this file to set environment variables for your local setup
+#
+EMUS_SCRAPER_USER=""
+EMUS_SCRAPER_PASS=""
+EMUS_JSON_LOGGING=True

+ 89 - 15
emus/settings.py

@@ -1,16 +1,16 @@
-"""
-Django settings for emus project.
-
-Generated by 'django-admin startproject' using Django 3.1.2.
+import os
 
-For more information on this file, see
-https://docs.djangoproject.com/en/3.1/topics/settings/
+from django.utils.translation import gettext_lazy as _
+from dotenv import load_dotenv
 
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.1/ref/settings/
-"""
+# Tap emus.conf if it's available
+if os.path.exists("emus.conf"):
+    load_dotenv("emus.conf")
+elif os.path.exists("/etc/emus.conf"):
+    load_dotenv("/etc/emus.conf")
+elif os.path.exists("/usr/local/etc/emus.conf"):
+    load_dotenv("/usr/local/etc/emus.conf")
 
-import os
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -126,9 +126,83 @@ MEDIA_ROOT = os.path.join(BASE_DIR, "media")
 ROMS_DIR = os.path.join(MEDIA_ROOT, "roms")
 
 SCRAPER_CONFIG = {
-    "site": os.getenv("SCRAPER_SITE", "screenscraper"),
-    "user": os.getenv("SCRAPER_USER", None),
-    "pass": os.getenv("SCRAPER_PASS", None),
-    "threads": os.getenv("SCRAPER_THREADS", 2),
-    "bin_path": os.getenv("SCRAPER_BINPATH", "Skyscraper"),
+    "site": os.getenv("EMUS_SCRAPER_SITE", "screenscraper"),
+    "user": os.getenv("EMUS_SCRAPER_USER", None),
+    "pass": os.getenv("EMUS_SCRAPER_PASS", None),
+    "threads": os.getenv("EMUS_SCRAPER_THREADS", 2),
+    "bin_path": os.getenv("EMUS_SCRAPER_BINPATH", "Skyscraper"),
+}
+
+JSON_LOGGING = os.getenv("EMUS_JSON_LOGGING", False)
+LOG_TYPE = "json" if JSON_LOGGING else "log"
+
+default_level = "INFO"
+if DEBUG:
+    default_level = "DEBUG"
+
+LOG_LEVEL = os.getenv("EMUS_LOG_LEVEL", default_level)
+LOG_FILE_PATH = os.getenv("EMUS_LOG_FILE_PATH", "/tmp/")
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "root": {"handlers": ["console", "file"], "level": LOG_LEVEL, "propagate": True},
+    "formatters": {
+        "color": {
+            "()": "colorlog.ColoredFormatter",
+            # \r returns caret to line beginning, in tests this eats the silly dot that removes
+            # the beautiful alignment produced below
+            "format": "\r"
+            "{log_color}{levelname:8s}{reset} "
+            "{bold_cyan}{name}{reset}:"
+            "{fg_bold_red}{lineno}{reset} "
+            "{thin_yellow}{funcName} "
+            "{thin_white}{message}"
+            "{reset}",
+            "style": "{",
+        },
+        "log": {"format": "%(asctime)s %(levelname)s %(message)s"},
+        "json": {
+            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",
+            "format": "%(levelname)s %(name) %(funcName) %(lineno) %(asctime)s %(message)s",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "color",
+            "level": LOG_LEVEL,
+        },
+        "null": {
+            "class": "logging.NullHandler",
+            "level": LOG_LEVEL,
+        },
+        "file": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "".join([LOG_FILE_PATH, "emus.", LOG_TYPE]),
+            "formatter": LOG_TYPE,
+            "level": LOG_LEVEL,
+        },
+        "requests_file": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "".join([LOG_FILE_PATH, "emus_requests.", LOG_TYPE]),
+            "formatter": LOG_TYPE,
+            "level": LOG_LEVEL,
+        },
+    },
+    "loggers": {
+        # Quiet down our console a little
+        "django.channels.server": {
+            "handlers": ["requests_file", "console"],
+            "level": "ERROR",
+            "propagate": False,
+        },
+        "django": {
+            "handlers": ["file"],
+            "propagate": True,
+        },
+        "daphne": {"handlers": ["file"], "propagate": False},
+        "django.db.backends": {"handlers": ["null"]},
+        "emus": {"handlers": ["console", "file"], "propagate": True},
+    },
 }

+ 58 - 1
poetry.lock

@@ -20,6 +20,28 @@ python-versions = ">=3.6"
 [package.extras]
 tzdata = ["tzdata"]
 
+[[package]]
+name = "colorama"
+version = "0.4.4"
+description = "Cross-platform colored terminal text."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[[package]]
+name = "colorlog"
+version = "6.6.0"
+description = "Add colours to the output of Python's logging module."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+
+[package.extras]
+development = ["black", "flake8", "mypy", "pytest", "types-colorama"]
+
 [[package]]
 name = "django"
 version = "4.0.3"
@@ -60,6 +82,25 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "python-dotenv"
+version = "0.20.0"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
+[[package]]
+name = "python-json-logger"
+version = "2.0.2"
+description = "A python library adding a json log formatter"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -87,7 +128,7 @@ python-versions = ">=2"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "940ba0ba20765fcece803e2a2db780aa96866fb798add03c6987ce919d92d816"
+content-hash = "020108f8e675d6fcf2e0b315047a7dc6f57bd4f3e9bb8f9cf629b1b666583e67"
 
 [metadata.files]
 asgiref = [
@@ -112,6 +153,14 @@ asgiref = [
     {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"},
     {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"},
 ]
+colorama = [
+    {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
+    {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
+]
+colorlog = [
+    {file = "colorlog-6.6.0-py2.py3-none-any.whl", hash = "sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e"},
+    {file = "colorlog-6.6.0.tar.gz", hash = "sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8"},
+]
 django = [
     {file = "Django-4.0.3-py3-none-any.whl", hash = "sha256:1239218849e922033a35d2a2f777cb8bee18bd725416744074f455f34ff50d0c"},
     {file = "Django-4.0.3.tar.gz", hash = "sha256:77ff2e7050e3324c9b67e29b6707754566f58514112a9ac73310f60cd5261930"},
@@ -124,6 +173,14 @@ python-dateutil = [
     {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
     {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
 ]
+python-dotenv = [
+    {file = "python-dotenv-0.20.0.tar.gz", hash = "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f"},
+    {file = "python_dotenv-0.20.0-py3-none-any.whl", hash = "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"},
+]
+python-json-logger = [
+    {file = "python-json-logger-2.0.2.tar.gz", hash = "sha256:202a4f29901a4b8002a6d1b958407eeb2dd1d83c18b18b816f5b64476dde9096"},
+    {file = "python_json_logger-2.0.2-py3-none-any.whl", hash = "sha256:99310d148f054e858cd5f4258794ed6777e7ad2c3fd7e1c1b527f1cba4d08420"},
+]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},

+ 3 - 0
pyproject.toml

@@ -9,6 +9,9 @@ python = "^3.8"
 Django = "^4.0.3"
 django-extensions = "^3.1.5"
 python-dateutil = "^2.8.2"
+python-dotenv = "^0.20.0"
+python-json-logger = "^2.0.2"
+colorlog = "^6.6.0"
 
 [tool.poetry.dev-dependencies]