Update importing and scraping utils

.gitignore

@@ -1,3 +1,4 @@
 db.sqlite3
 emus.conf
 /media/
+skyscraper.ini

emus/settings.py

@@ -133,13 +133,9 @@ MEDIA_URL = "/media/"
 MEDIA_ROOT = os.getenv("EMUS_MEDIA_ROOT", os.path.join(BASE_DIR, "media"))
 ROMS_DIR = os.path.join(MEDIA_ROOT, "roms")
 
-SCRAPER_CONFIG = {
-    "site": os.getenv("EMUS_SCRAPER_SITE", "screenscraper"),
-    "user": os.getenv("EMUS_SCRAPER_USER", None),
-    "pass": os.getenv("EMUS_SCRAPER_PASS", None),
-    "threads": os.getenv("EMUS_SCRAPER_THREADS", 2),
-    "bin_path": os.getenv("EMUS_SCRAPER_BINPATH", "Skyscraper"),
-}
+SCRAPER_BIN_PATH = os.getenv("EMUS_SCRAPER_BINPATH", "Skyscraper")
+SCRAPER_CONFIG_FILE = os.getenv("EMUS_SCRAPER_CONFIG_FILE", "skyscraper.ini")
+SCRAPER_SITE = os.getenv("EMUS_SCRAPER_SITE", "screenscraper")
 
 JSON_LOGGING = os.getenv("EMUS_JSON_LOGGING", False)
 LOG_TYPE = "json" if JSON_LOGGING else "log"
@@ -215,6 +211,8 @@ LOGGING = {
     },
 }
 
+REMOVE_FROM_SLUGS = ["_", " ", "/"]
+
 if DEBUG:
     # We clear out a db with lots of games all the time in dev
     DATA_UPLOAD_MAX_NUMBER_FIELDS = 3000

games/models.py

@@ -1,5 +1,6 @@
 from enum import Enum
 
+from django.conf import settings
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.urls import reverse
@@ -39,7 +40,9 @@ class BaseModel(models.Model):
         abstract = True
 
     def slugify_function(self, content):
-        return content.replace("_", "-").replace(" ", "-").lower()
+        for element in settings.REMOVE_FROM_SLUGS:
+            content = content.replace(element, "-")
+        return content.lower()
 
     def __str__(self):
         return self.name

games/utils.py

@@ -18,8 +18,10 @@ EU_STRINGS = ["(e)", "eur", "europe", "pal"]
 
 def update_media_root_for_import(file_path):
     """Given a file path, re-write it for our app MEDIA_ROOT"""
-    split_path = file_path.split("/media/")
-    return split_path[-1]
+    suffix = ""
+    if file_path:
+        suffix = file_path.split("/media/")[-1]
+    return suffix
 
 
 def import_gamelist_file_to_db_for_system(game_system_slug, file_path=None):
@@ -88,7 +90,10 @@ def import_gamelist_file_to_db_for_system(game_system_slug, file_path=None):
         description = game.find("desc").text
         screenshot_path = update_media_root_for_import(game.find("image").text)
         rom_path = update_media_root_for_import(game.find("path").text)
-        video_path = update_media_root_for_import(game.find("video").text)
+        video_path_elem = game.find("video")
+        video_path = ""
+        if video_path_elem:
+            video_path = update_media_root_for_import(video_path_elem.text)
         marquee_path = update_media_root_for_import(game.find("marquee").text)
 
         obj, created = Game.objects.get_or_create(name=name)
@@ -166,26 +171,29 @@ def export_gamelist_file_to_path_for_system(game_system_slug, file_path=None):
 
 
 def skyscrape_console(game_system_slug):
-    scraper_config = settings.SCRAPER_CONFIG
-    binary = scraper_config["bin_path"]
-    site = scraper_config["site"]
-    user = scraper_config["user"]
-    password = scraper_config["pass"]
-    threads = scraper_config["threads"]
-
-    scrape_cmd = f"{binary} -s {site} -u {user}:{password} -t {threads} -f emulationstation -p {game_system_slug}"
-    load_cmd = f"{binary} -f emulationstation -p {game_system_slug}"
+    scraper_config = settings.SCRAPER_CONFIG_FILE
+    scraper_binary = settings.SCRAPER_BIN_PATH
+    scraper_site = settings.SCRAPER_SITE
+
+    # If the config file is relative, append our base dir
+    print(f"Preparing to scrape game info for {game_system_slug}")
+    if scraper_config[0] != "/":
+        scraper_config = os.path.join(settings.BASE_DIR, scraper_config)
+    if not os.path.exists(scraper_config):
+        logger.info(f"Config file not found at {scraper_config}")
+        print(f"Config file not found at {scraper_config}")
+        return
+    print(f"Using configuration file from {scraper_config}")
+
+    # scrape_cmd = f"{binary} -c {config_file} -s {site} -u {user}:{password} -t {threads} -f emulationstation -p {game_system_slug}"
+    # load_cmd = f"{binary} -f emulationstation -p {game_system_slug}"
     scrape_output = subprocess.run(
         [
-            binary,
+            scraper_binary,
+            "-c",
+            f"{scraper_config}",
             "-s",
-            "{site}",
-            "-u",
-            f"{user}:{password}",
-            "-t",
-            f"{threads}",
-            "-f",
-            "emulationstation",
+            f"{scraper_site}",
             "-p",
             f"{game_system_slug}",
         ],
@@ -193,12 +201,16 @@ def skyscrape_console(game_system_slug):
     )
     load_output = subprocess.run(
         [
-            binary,
+            scraper_binary,
+            "-c",
+            f"{scraper_config}",
             "-f",
-            "emulationstation",
+            "{scraper_site}",
             "-p",
             f"{game_system_slug}",
         ],
         capture_output=True,
     )
+    print(scrape_output)
+    print(load_output)
     return scrape_output, load_output

skyscraper.ini.example

@@ -0,0 +1,35 @@
+; --------------------------------------------------------------------
+; Skyscraper by Lars Muldjord (https://github.com/muldjord/skyscraper)
+; --------------------------------------------------------------------
+
+[main]
+inputFolder="/media/roms"
+mediaFolder="/media/roms"
+gameListFolder="/media/roms"
+cacheFolder="media/roms/.skyscraper/cache"
+cacheCovers="true"
+cacheScreenshots="true"
+cacheWheels="true"
+cacheMarquees="true"
+cacheResize="false"
+nameTemplate="%t"
+jpgQuality="85"
+unattend="true"
+interactive="false"
+forceFilename="true"
+threads="2"
+unpac="false"
+videos="false"
+frontend="emulationstation"
+
+;[screenscraper]
+;userCreds="user:pass"
+
+[neogeo]
+nameTemplate="%t [%f]"
+
+[esgamelist]
+cacheRefresh="true"
+
+[import]
+cacheRefresh="true"

templates/games/gamesystem_detail.html

@@ -7,8 +7,8 @@
     {% for  game in object.game_set.all %}
         <div class="card d-flex float-left" style="width: 12em; padding:1em">
         <a href="{{game.get_absolute_url}}">
-            {% if game.marquee %}
-            <img class="card-img-top" src="{{game.marquee.url}}" alt="Card image cap">
+            {% if game.screenshot %}
+            <img class="card-img-top" src="{{game.screenshot.url}}" alt="Card image cap">
             {% else %}
             <i>{{game.name}}</i>
             {% endif %}