Add retroarch import tasks and models

vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-05-25 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0008_userprofile_lastfm_auto_import"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="retroarch_auto_import",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="retroarch_path",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/profiles/models.py

@@ -25,6 +25,9 @@ class UserProfile(TimeStampedModel):
     lastfm_password = EncryptedField(**BNULL)
     lastfm_auto_import = models.BooleanField(default=False)
 
+    retroarch_path = models.CharField(max_length=255, **BNULL)
+    retroarch_auto_import = models.BooleanField(default=False)
+
     def __str__(self):
         return f"User profile for {self.user}"
 

vrobbler/apps/scrobbles/admin.py

@@ -5,6 +5,7 @@ from scrobbles.models import (
     ChartRecord,
     KoReaderImport,
     LastFmImport,
+    RetroarchImport,
     Scrobble,
 )
 from scrobbles.mixins import Genre
@@ -38,17 +39,22 @@ class ImportBaseAdmin(admin.ModelAdmin):
 
 @admin.register(AudioScrobblerTSVImport)
 class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
 
 
 @admin.register(LastFmImport)
 class LastFmImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
 
 
 @admin.register(KoReaderImport)
 class KoReaderImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
+
+
+@admin.register(RetroarchImport)
+class RetroarchImportAdmin(ImportBaseAdmin):
+    ...
 
 
 @admin.register(Genre)

vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--restart",
+            action="store_true",
+            help="Restart failed imports",
+        )
+
+    def handle(self, *args, **options):
+        restart = False
+        if options["restart"]:
+            restart = True
+        count = import_retroarch_for_all_users(restart=restart)
+        print(f"Started {count} Retroarch imports")

vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py

@@ -0,0 +1,67 @@
+# Generated by Django 4.1.7 on 2023-05-25 02:17
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RetroarchImport",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
+                (
+                    "processing_started",
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                (
+                    "processed_finished",
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                ("process_log", models.TextField(blank=True, null=True)),
+                ("process_count", models.IntegerField(blank=True, null=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Retroarch Import",
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -30,6 +30,7 @@ from scrobbles.utils import (
     check_scrobble_for_finish,
 )
 from sports.models import SportEvent
+from videogames import retroarch
 from videogames.models import VideoGame
 from videos.models import Series, Video
 
@@ -50,7 +51,7 @@ class BaseFileImportMixin(TimeStampedModel):
         abstract = True
 
     def __str__(self):
-        return f"Scrobble import {self.id}"
+        return f"{self.import_type} import on {self.human_start}"
 
     @property
     def human_start(self):
@@ -144,9 +145,6 @@ class KoReaderImport(BaseFileImportMixin):
     def import_type(self) -> str:
         return "KOReader"
 
-    def __str__(self):
-        return f"KoReader import on {self.human_start}"
-
     def get_absolute_url(self):
         return reverse(
             "scrobbles:koreader-import-detail", kwargs={"slug": self.uuid}
@@ -192,9 +190,6 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
     def import_type(self) -> str:
         return "AudiosScrobbler"
 
-    def __str__(self):
-        return f"Audioscrobbler import on {self.human_start}"
-
     def get_absolute_url(self):
         return reverse(
             "scrobbles:tsv-import-detail", kwargs={"slug": self.uuid}
@@ -246,9 +241,6 @@ class LastFmImport(BaseFileImportMixin):
     def import_type(self) -> str:
         return "LastFM"
 
-    def __str__(self):
-        return f"LastFM import on {self.human_start}"
-
     def get_absolute_url(self):
         return reverse(
             "scrobbles:lastfm-import-detail", kwargs={"slug": self.uuid}
@@ -288,6 +280,46 @@ class LastFmImport(BaseFileImportMixin):
         self.mark_finished()
 
 
+class RetroarchImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "Retroarch Import"
+
+    @property
+    def import_type(self) -> str:
+        return "Retroarch"
+
+    def get_absolute_url(self):
+        return reverse(
+            "scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid}
+        )
+
+    def process(self, import_all=False, force=False):
+        """Import scrobbles found on Retroarch"""
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        if force:
+            logger.info(f"You told me to force import from Retroarch")
+
+        if not self.user.profile.retroarch_path:
+            logger.info(
+                "Tying to import Retroarch logs, but user has no retroarch_path configured"
+            )
+
+        self.mark_started()
+
+        scrobbles = retroarch.import_retroarch_lrtl_files(
+            self.user.profile.retroarch_path,
+            self.user.id,
+        )
+
+        self.record_log(scrobbles)
+        self.mark_finished()
+
+
 class ChartRecord(TimeStampedModel):
     """Sort of like a materialized view for what we could dynamically generate,
     but would kill the DB as it gets larger. Collects time-based records

vrobbler/apps/scrobbles/tasks.py

@@ -9,6 +9,16 @@ logger = logging.getLogger(__name__)
 User = get_user_model()
 
 
+@shared_task
+def process_retroarch_import(import_id):
+    RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
+    retroarch_import = RetroarchImport.objects.filter(id=import_id).first()
+    if not retroarch_import:
+        logger.warn(f"RetroarchImport not found with id {import_id}")
+
+    retroarch_import.process()
+
+
 @shared_task
 def process_lastfm_import(import_id):
     LastFmImport = apps.get_model("scrobbles", "LastFMImport")

vrobbler/apps/scrobbles/urls.py

@@ -65,6 +65,11 @@ urlpatterns = [
         views.ScrobbleKoReaderImportDetailView.as_view(),
         name="koreader-import-detail",
     ),
+    path(
+        "imports/retroarch/<slug:slug>/",
+        views.ScrobbleRetroarchImportDetailView.as_view(),
+        name="retroarch-import-detail",
+    ),
     path(
         "charts/",
         views.ChartRecordView.as_view(),

vrobbler/apps/scrobbles/utils.py

@@ -11,7 +11,7 @@ from django.utils import timezone
 from profiles.models import UserProfile
 from profiles.utils import now_user_timezone
 from scrobbles.constants import LONG_PLAY_MEDIA
-from scrobbles.tasks import process_lastfm_import
+from scrobbles.tasks import process_lastfm_import, process_retroarch_import
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
@@ -208,6 +208,30 @@ def import_lastfm_for_all_users(restart=False):
     return lastfm_import_count
 
 
+def import_retroarch_for_all_users(restart=False):
+    """Grab a list of all users with Retroarch enabled and kickoff imports for them"""
+    RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
+    retroarch_enabled_user_ids = UserProfile.objects.filter(
+        retroarch_path__isnull=False,
+        retroarch_auto_import=True,
+    ).values_list("user_id", flat=True)
+
+    retroarch_import_count = 0
+
+    for user_id in retroarch_enabled_user_ids:
+        retroarch_import, created = RetroarchImport.objects.get_or_create(
+            user_id=user_id, processed_finished__isnull=True
+        )
+        if not created and not restart:
+            logger.info(
+                f"Not resuming failed LastFM import {retroarch_import.id} for user {user_id}, use restart=True to restart"
+            )
+            continue
+        process_retroarch_import.delay(retroarch_import.id)
+        retroarch_import_count += 1
+    return retroarch_import_count
+
+
 def delete_zombie_scrobbles(dry_run=True):
     """Look for any scrobble over a day old that is not paused and still in progress and delete it"""
     Scrobble = apps.get_model("scrobbles", "Scrobble")

vrobbler/apps/scrobbles/views.py

@@ -9,7 +9,6 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
-from django.db.models.fields import timezone
 from django.db.models.query import QuerySet
 from django.http import FileResponse, HttpResponseRedirect, JsonResponse
 from django.urls import reverse, reverse_lazy
@@ -42,16 +41,17 @@ from scrobbles.models import (
     ChartRecord,
     KoReaderImport,
     LastFmImport,
+    RetroarchImport,
     Scrobble,
 )
 from scrobbles.scrobblers import (
     jellyfin_scrobble_track,
     jellyfin_scrobble_video,
+    manual_scrobble_board_game,
     manual_scrobble_book,
     manual_scrobble_event,
     manual_scrobble_video,
     manual_scrobble_video_game,
-    manual_scrobble_board_game,
     mopidy_scrobble_podcast,
     mopidy_scrobble_track,
 )
@@ -202,6 +202,10 @@ class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView):
     model = LastFmImport
 
 
+class ScrobbleRetroarchImportDetailView(BaseScrobbleImportDetailView):
+    model = RetroarchImport
+
+
 class ManualScrobbleView(FormView):
     form_class = ScrobbleForm
     template_name = "scrobbles/manual_form.html"

vrobbler/apps/videogames/retroarch.py

@@ -6,6 +6,7 @@ from typing import List
 
 import pytz
 from dateutil.parser import ParserError, parse
+from django.apps import apps
 
 from vrobbler.apps.scrobbles.utils import convert_to_seconds
 from vrobbler.apps.videogames.utils import get_or_create_videogame
@@ -71,6 +72,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]:
         3. Create new scrobble if last_played != last_scrobble.timestamp
         4. Calculate scrobble time from runtime - last_scrobble.long_play_time
     """
+    Scrobble = apps.get_model("scrobbles", "Scrobble")
 
     game_logs = load_game_data(playlog_path)
     found_game = None
@@ -104,18 +106,23 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]:
             stop_timestamp = game_data["last_played"] + timedelta(
                 seconds=playback_position_seconds
             )
+            if playback_position_seconds < 30:
+                logger.info(
+                    f"Video game {found_game.id} played for less than 30 seconds, skipping"
+                )
             new_scrobbles.append(
-                {
-                    "video_game_id": found_game.id,
-                    "timestamp": game_data["last_played"],
-                    "stop_timestamp": stop_timestamp,
-                    "playback_position_seconds": playback_position_seconds,
-                    "played_to_completion": True,
-                    "in_progress": False,
-                    "long_play_seconds": game_data["runtime"],
-                    "user_id": user_id,
-                    "source_id": "Retroarch",
-                    "source": "Imported from Retroarch play log file",
-                }
+                Scrobble(
+                    video_game_id=found_game.id,
+                    timestamp=game_data["last_played"],
+                    stop_timestamp=stop_timestamp,
+                    playback_position_seconds=playback_position_seconds,
+                    played_to_completion=True,
+                    in_progress=False,
+                    long_play_seconds=game_data["runtime"],
+                    user_id=user_id,
+                    source="Retroarch",
+                    source_id="Imported from Retroarch play log file",
+                )
             )
+    created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles)
     return new_scrobbles