[importers] Reorganize importers a little

vrobbler/apps/scrobbles/imap.py → vrobbler/apps/scrobbles/importers/imap.py

@@ -1,16 +1,17 @@
-import json
-import imaplib
 import email
+import imaplib
+import json
+import logging
 from email.header import decode_header
+
 from profiles.models import UserProfile
 from scrobbles.models import Scrobble
 from scrobbles.scrobblers import email_scrobble_board_game
-import logging
 
 logger = logging.getLogger(__name__)
 
 
-def process_scrobbles_from_imap() -> list[Scrobble]:
+def import_scrobbles_from_imap() -> list[Scrobble]:
     """For all user profiles with IMAP creds, check inbox for scrobbleable email attachments."""
     scrobbles_created: list[Scrobble] = []
 

vrobbler/apps/scrobbles/importers/webdav.py

@@ -0,0 +1,81 @@
+from books.koreader import fetch_file_from_webdav
+from profiles.models import UserProfile
+from scrobbles.models import KoReaderImport
+from scrobbles.tasks import process_koreader_import
+from webdav.client import get_webdav_client
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def import_from_webdav_for_all_users(restart=False):
+    """Grab a list of all users with WebDAV enabled and kickoff imports for them"""
+
+    # LastFmImport = apps.get_model("scrobbles", "LastFMImport")
+    webdav_enabled_user_ids = UserProfile.objects.filter(
+        webdav_url__isnull=False,
+        webdav_user__isnull=False,
+        webdav_pass__isnull=False,
+        webdav_auto_import=True,
+    ).values_list("user_id", flat=True)
+    logger.info(
+        f"Start import of {webdav_enabled_user_ids.count()} webdav accounts"
+    )
+
+    koreader_import_count = 0
+
+    for user_id in webdav_enabled_user_ids:
+        webdav_client = get_webdav_client(user_id)
+
+        try:
+            webdav_client.info("var/koreader/statistics.sqlite3")
+            koreader_found = True
+        except:
+            koreader_found = False
+            logger.info(
+                "No koreader stats file found on webdav",
+                extra={"user_id": user_id},
+            )
+
+        if koreader_found:
+            last_import = (
+                KoReaderImport.objects.filter(
+                    user_id=user_id, processed_finished__isnull=False
+                )
+                .order_by("Processed_finished")
+                .last()
+            )
+
+            koreader_file_path = fetch_file_from_webdav(1)
+            new_hash = get_file_md5_hash(koreader_file_path)
+            old_hash = None
+            if last_import:
+                old_hash = last_import.file_md5_hash()
+
+            if old_hash and new_hash == old_hash:
+                logger.info(
+                    "koreader stats file has not changed",
+                    extra={
+                        "user_id": user_id,
+                        "new_hash": new_hash,
+                        "old_hash": old_hash,
+                        "last_import_id": last_import.id,
+                    },
+                )
+                continue
+
+            koreader_import, created = KoReaderImport.objects.get_or_create(
+                user_id=user_id, processed_finished__isnull=True
+            )
+
+            if not created and not restart:
+                logger.info(
+                    f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart"
+                )
+                continue
+
+            koreader_import.save_sqlite_file_to_self(koreader_file_path)
+
+            process_koreader_import.delay(koreader_import.id)
+            koreader_import_count += 1
+    return koreader_import_count

vrobbler/apps/scrobbles/management/commands/import_from_imap.py

@@ -1,5 +1,5 @@
 from django.core.management.base import BaseCommand
-from vrobbler.apps.scrobbles.imap import process_scrobbles_from_imap
+from vrobbler.apps.scrobbles.importers.imap import import_scrobbles_from_imap
 
 
 class Command(BaseCommand):
@@ -11,5 +11,5 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
-        count = len(process_scrobbles_from_imap())
+        count = len(import_scrobbles_from_imap())
         print(f"Started {count} IMAP imports")

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -1,5 +1,5 @@
 from django.core.management.base import BaseCommand
-from vrobbler.apps.scrobbles.utils import import_from_webdav_for_all_users
+from vrobbler.apps.scrobbles.importers import webdav
 
 
 class Command(BaseCommand):
@@ -14,5 +14,5 @@ class Command(BaseCommand):
         restart = False
         if options["restart"]:
             restart = True
-        count = import_from_webdav_for_all_users(restart=restart)
+        count = webdav.import_from_webdav_for_all_users(restart=restart)
         print(f"Started {count} WeDAV imports")