Add book scrobbling

vrobbler/apps/books/admin.py

@@ -0,0 +1,25 @@
+from django.contrib import admin
+
+from books.models import Author, Book
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Author)
+class AlbumAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "openlibrary_id")
+    ordering = ("name",)
+
+
+@admin.register(Book)
+class ArtistAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "isbn",
+        "first_publish_year",
+        "pages",
+        "openlibrary_id",
+    )
+    ordering = ("title",)

vrobbler/apps/books/migrations/0001_initial.py

@@ -0,0 +1,128 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:17
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Author',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'run_time',
+                    models.CharField(blank=True, max_length=8, null=True),
+                ),
+                (
+                    'run_time_ticks',
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ('title', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'goodreads_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('koreader_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'koreader_authors',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'koreader_md5',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'isbn',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('pages', models.IntegerField(blank=True, null=True)),
+                (
+                    'language',
+                    models.CharField(blank=True, max_length=4, null=True),
+                ),
+                (
+                    'first_publish_year',
+                    models.IntegerField(blank=True, null=True),
+                ),
+                ('authors', models.ManyToManyField(to='books.author')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Dict
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import ScrobblableMixin
+
+from vrobbler.apps.books.utils import lookup_book_from_openlibrary
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+BNULL = {"blank": True, "null": True}
+
+
+class Author(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def fix_metadata(self):
+        logger.warn("Not implemented yet")
+
+
+class Book(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'BOOK_COMPLETION_PERCENT', 95)
+
+    title = models.CharField(max_length=255)
+    authors = models.ManyToManyField(Author)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    koreader_id = models.IntegerField(**BNULL)
+    koreader_authors = models.CharField(max_length=255, **BNULL)
+    koreader_md5 = models.CharField(max_length=255, **BNULL)
+    isbn = models.CharField(max_length=255, **BNULL)
+    pages = models.IntegerField(**BNULL)
+    language = models.CharField(max_length=4, **BNULL)
+    first_publish_year = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        return f"{self.title} by {self.author}"
+
+    def fix_metadata(self):
+        if not self.openlibrary_id:
+            book_meta = lookup_book_from_openlibrary(self.title, self.author)
+            self.openlibrary_id = book_meta.get("openlibrary_id")
+            self.isbn = book_meta.get("isbn")
+            self.goodreads_id = book_meta.get("goodreads_id")
+            self.first_pubilsh_year = book_meta.get("first_publish_year")
+            self.save()
+
+    @property
+    def author(self):
+        return self.authors.first()
+
+    def get_absolute_url(self):
+        return reverse("books:book_detail", kwargs={'slug': self.uuid})
+
+    @property
+    def pages_for_completion(self) -> int:
+        if not self.pages:
+            logger.warn(f"{self} has no pages, no completion percentage")
+            return 0
+        return int(self.pages * (self.COMPLETION_PERCENT / 100))

vrobbler/apps/books/utils.py

@@ -0,0 +1,47 @@
+import json
+from typing import Optional
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+SEARCH_URL = "https://openlibrary.org/search.json?title={title}"
+ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
+    search_url = SEARCH_URL.format(title=title)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if len(results.get('docs')) == 0:
+        logger.warn(f"No results found from OL for {title}")
+        return {}
+
+    top = results.get('docs')[0]
+    if author and author not in top['author_name']:
+        logger.warn(
+            f"Lookup for {title} found top result with mismatched author"
+        )
+
+    return {
+        "title": top.get("title"),
+        "isbn": top.get("isbn")[0],
+        "openlibrary_id": top.get("cover_edition_key"),
+        "author_name": get_first("author_name", top),
+        "author_openlibrary_id": get_first("author_key", top),
+        "goodreads_id": get_first("id_goodreads", top),
+        "first_publish_year": top.get("first_publish_year"),
+    }

vrobbler/apps/music/utils.py

@@ -1,5 +1,5 @@
-import re
 import logging
+import re
 
 from scrobbles.musicbrainz import (
     lookup_album_dict_from_mb,
@@ -9,7 +9,7 @@ from scrobbles.musicbrainz import (
 logger = logging.getLogger(__name__)
 
 
-from music.models import Artist, Album, Track
+from music.models import Album, Artist, Track
 
 
 def get_or_create_artist(name: str, mbid: str = None) -> Artist:

vrobbler/apps/profiles/models.py

@@ -16,7 +16,7 @@ class UserProfile(TimeStampedModel):
         User, on_delete=models.CASCADE, related_name="profile"
     )
     timezone = models.CharField(
-        max_length=255, choices=PRETTY_TIMEZONE_CHOICES
+        max_length=255, choices=PRETTY_TIMEZONE_CHOICES, default=pytz.UTC
     )
     lastfm_username = models.CharField(max_length=255, **BNULL)
     lastfm_password = EncryptedField(**BNULL)

vrobbler/apps/scrobbles/admin.py

@@ -2,6 +2,7 @@ from django.contrib import admin
 from scrobbles.models import (
     AudioScrobblerTSVImport,
     ChartRecord,
+    KoReaderImport,
     LastFmImport,
     Scrobble,
 )
@@ -14,15 +15,7 @@ class ScrobbleInline(admin.TabularInline):
     exclude = ('source_id', 'scrobble_log')
 
 
-@admin.register(AudioScrobblerTSVImport)
-class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
-    date_hierarchy = "created"
-    list_display = ("uuid", "created", "process_count", "tsv_file")
-    ordering = ("-created",)
-
-
-@admin.register(LastFmImport)
-class LastFmImportAdmin(admin.ModelAdmin):
+class ImportBaseAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "uuid",
@@ -33,6 +26,21 @@ class LastFmImportAdmin(admin.ModelAdmin):
     ordering = ("-created",)
 
 
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(LastFmImport)
+class LastFmImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(KoReaderImport)
+class KoReaderImportAdmin(ImportBaseAdmin):
+    """"""
+
+
 @admin.register(ChartRecord)
 class ChartRecordAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
@@ -71,21 +79,21 @@ class ScrobbleAdmin(admin.ModelAdmin):
         "is_paused",
         "played_to_completion",
     )
-    raw_id_fields = ('video', 'podcast_episode', 'track', 'sport_event')
+    raw_id_fields = (
+        'video',
+        'podcast_episode',
+        'track',
+        'sport_event',
+        'book',
+    )
     list_filter = ("is_paused", "in_progress", "source", "track__artist")
     ordering = ("-timestamp",)
 
     def media_name(self, obj):
-        if obj.video:
-            return obj.video
-        if obj.track:
-            return obj.track
-        if obj.podcast_episode:
-            return obj.podcast_episode
-        if obj.sport_event:
-            return obj.sport_event
+        return obj.media_obj
 
     def media_type(self, obj):
+        return obj.media_obj.__class__.__name__
         if obj.video:
             return "Video"
         if obj.track:

vrobbler/apps/scrobbles/koreader.py

@@ -0,0 +1,124 @@
+import logging
+from datetime import datetime
+import sqlite3
+from enum import Enum
+
+import pytz
+
+from books.models import Author, Book
+from scrobbles.models import Scrobble
+from django.utils import timezone
+
+logger = logging.getLogger(__name__)
+
+
+class KoReaderBookColumn(Enum):
+    ID = 0
+    TITLE = 1
+    AUTHORS = 2
+    NOTES = 3
+    LAST_OPEN = 4
+    HIGHLIGHTS = 5
+    PAGES = 6
+    SERIES = 7
+    LANGUAGE = 8
+    MD5 = 9
+    TOTAL_READ_TIME = 10
+    TOTAL_READ_PAGES = 11
+
+
+class KoReaderPageStatColumn(Enum):
+    ID_BOOK = 0
+    PAGE = 1
+    START_TIME = 2
+    DURATION = 3
+    TOTAL_PAGES = 4
+
+
+def process_koreader_sqlite_file(sqlite_file_path, user_id):
+    """Given a sqlite file from KoReader, open the book table, iterate
+    over rows creating scrobbles from each book found"""
+    # Create a SQL connection to our SQLite database
+    con = sqlite3.connect(sqlite_file_path)
+    cur = con.cursor()
+
+    # Return all results of query
+    book_table = cur.execute("SELECT * FROM book")
+    new_scrobbles = []
+    for book_row in book_table:
+        authors = book_row[KoReaderBookColumn.AUTHORS.value].split('\n')
+        author_list = []
+        for author_str in authors:
+            logger.debug(f"Looking up author {author_str}")
+
+            if author_str == "N/A":
+                continue
+
+            author, created = Author.objects.get_or_create(name=author_str)
+            if created:
+                author.fix_metadata()
+            author_list.append(author)
+            logger.debug(f"Found author {author}, created: {created}")
+
+        book, created = Book.objects.get_or_create(
+            koreader_md5=book_row[KoReaderBookColumn.MD5.value]
+        )
+
+        if created:
+            book.title = book_row[KoReaderBookColumn.TITLE.value]
+            book.pages = book_row[KoReaderBookColumn.PAGES.value]
+            book.koreader_id = int(book_row[KoReaderBookColumn.ID.value])
+            book.koreader_authors = book_row[KoReaderBookColumn.AUTHORS.value]
+            book.run_time_ticks = int(book_row[KoReaderBookColumn.PAGES.value])
+            book.save(
+                update_fields=[
+                    "title",
+                    "pages",
+                    "koreader_id",
+                    "koreader_authors",
+                ]
+            )
+            book.fix_metadata()
+            if author_list:
+                book.authors.add(*[a.id for a in author_list])
+
+        playback_position = int(
+            book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]
+        )
+        playback_position_ticks = playback_position * 1000
+        pages_read = int(book_row[KoReaderBookColumn.TOTAL_READ_PAGES.value])
+        timestamp = datetime.utcfromtimestamp(
+            book_row[KoReaderBookColumn.LAST_OPEN.value]
+        ).replace(tzinfo=pytz.utc)
+
+        new_scrobble = Scrobble(
+            book_id=book.id,
+            user_id=user_id,
+            source="KOReader",
+            timestamp=timestamp,
+            playback_position_ticks=playback_position_ticks,
+            playback_position=playback_position,
+            played_to_completion=True,
+            in_progress=False,
+            book_pages_read=pages_read,
+        )
+
+        existing = Scrobble.objects.filter(
+            timestamp=timestamp, book=book
+        ).first()
+        if existing:
+            logger.debug(f"Skipping existing scrobble {new_scrobble}")
+            continue
+
+        logger.debug(f"Queued scrobble {new_scrobble} for creation")
+        new_scrobbles.append(new_scrobble)
+
+    # Be sure to close the connection
+    con.close()
+
+    created = Scrobble.objects.bulk_create(new_scrobbles)
+    logger.info(
+        f"Created {len(created)} scrobbles",
+        extra={'created_scrobbles': created},
+    )
+    return created

vrobbler/apps/scrobbles/migrations/0020_alter_audioscrobblertsvimport_options_and_more.py

@@ -0,0 +1,97 @@
+# Generated by Django 4.1.5 on 2023-02-19 03:53
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import scrobbles.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        (
+            'scrobbles',
+            '0019_rename_processed_on_lastfmimport_processed_finished_and_more',
+        ),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='audioscrobblertsvimport',
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name='lastfmimport',
+            options={},
+        ),
+        migrations.RenameField(
+            model_name='audioscrobblertsvimport',
+            old_name='processed_on',
+            new_name='processed_finished',
+        ),
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='processing_started',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.CreateModel(
+            name='KoReaderImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)),
+                (
+                    'processing_started',
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                (
+                    'processed_finished',
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                ('process_log', models.TextField(blank=True, null=True)),
+                ('process_count', models.IntegerField(blank=True, null=True)),
+                (
+                    'sqlite_file',
+                    models.FileField(
+                        blank=True,
+                        null=True,
+                        upload_to=scrobbles.models.KoReaderImport.get_path,
+                    ),
+                ),
+                (
+                    'user',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0021_scrobble_book.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0001_initial'),
+        ('scrobbles', '0020_alter_audioscrobblertsvimport_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='book',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='books.book',
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0022_scrobble_book_pages_read.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-20 00:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0021_scrobble_book'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='book_pages_read',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -7,6 +7,7 @@ from django.db import models
 from django.utils import timezone
 from django_extensions.db.models import TimeStampedModel
 from music.models import Artist, Track
+from books.models import Book
 from podcasts.models import Episode
 from profiles.utils import now_user_timezone
 from scrobbles.lastfm import LastFM
@@ -19,71 +20,143 @@ User = get_user_model()
 BNULL = {"blank": True, "null": True}
 
 
-class AudioScrobblerTSVImport(TimeStampedModel):
-    def get_path(instance, filename):
-        extension = filename.split('.')[-1]
-        uuid = instance.uuid
-        return f'audioscrobbler-uploads/{uuid}.{extension}'
-
+class BaseFileImportMixin(TimeStampedModel):
     user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
     uuid = models.UUIDField(editable=False, default=uuid4)
-    tsv_file = models.FileField(upload_to=get_path, **BNULL)
-    processed_on = models.DateTimeField(**BNULL)
+    processing_started = models.DateTimeField(**BNULL)
+    processed_finished = models.DateTimeField(**BNULL)
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
 
+    class Meta:
+        abstract = True
+
     def __str__(self):
-        if self.tsv_file:
-            return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
-        return f"Audioscrobbler TSV upload {self.id}"
+        return f"Scrobble import {self.id}"
 
     def process(self, force=False):
-        from scrobbles.tsv import process_audioscrobbler_tsv_file
+        logger.warning("Process not implemented")
+
+    def undo(self, dryrun=False):
+        """Accepts the log from a scrobble import and removes the scrobbles"""
+        from scrobbles.models import Scrobble
 
-        if self.processed_on and not force:
-            logger.info(f"{self} already processed on {self.processed_on}")
+        if not self.process_log:
+            logger.warning("No lines in process log found to undo")
             return
 
-        tz = None
-        if self.user:
-            tz = self.user.profile.tzinfo
-        scrobbles = process_audioscrobbler_tsv_file(
-            self.tsv_file.path, self.user.id, user_tz=tz
+        for line in self.process_log.split('\n'):
+            scrobble_id = line.split("\t")[0]
+            scrobble = Scrobble.objects.filter(id=scrobble_id).first()
+            if not scrobble:
+                logger.warning(
+                    f"Could not find scrobble {scrobble_id} to undo"
+                )
+                continue
+            logger.info(f"Removing scrobble {scrobble_id}")
+            if not dryrun:
+                scrobble.delete()
+        self.processed_finished = None
+        self.processing_started = None
+        self.process_count = None
+        self.process_log = ""
+        self.save(
+            update_fields=[
+                "processed_finished",
+                "processing_started",
+                "process_log",
+                "process_count",
+            ]
         )
+
+    def mark_started(self):
+        self.processing_started = timezone.now()
+        self.save(update_fields=["processing_started"])
+
+    def mark_finished(self):
+        self.processed_finished = timezone.now()
+        self.save(update_fields=['processed_finished'])
+
+    def record_log(self, scrobbles):
         self.process_log = ""
-        if scrobbles:
-            for count, scrobble in enumerate(scrobbles):
-                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
-                log_line = f"{scrobble_str}"
-                if count > 0:
-                    log_line = "\n" + log_line
-                self.process_log += log_line
-            self.process_count = len(scrobbles)
-        else:
-            self.process_log = f"Created no new scrobbles"
+        if not scrobbles:
             self.process_count = 0
+            self.save(update_fields=["process_log" "process_count"])
+            return
 
-        self.processed_on = timezone.now()
-        self.save(
-            update_fields=['processed_on', 'process_count', 'process_log']
+        for count, scrobble in enumerate(scrobbles):
+            scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj.title}"
+            log_line = f"{scrobble_str}"
+            if count > 0:
+                log_line = "\n" + log_line
+            self.process_log += log_line
+        self.process_count = len(scrobbles)
+        self.save(update_fields=["process_log", "process_count"])
+
+
+class KoReaderImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "KOReader Import"
+
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'koreader-uploads/{uuid}.{extension}'
+
+    sqlite_file = models.FileField(upload_to=get_path, **BNULL)
+
+    def process(self, force=False):
+        from scrobbles.koreader import process_koreader_sqlite_file
+
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        self.mark_started()
+        scrobbles = process_koreader_sqlite_file(
+            self.sqlite_file.path, self.user.id
         )
+        self.record_log(scrobbles)
+        self.mark_finished()
 
-    def undo(self, dryrun=True):
-        from scrobbles.tsv import undo_audioscrobbler_tsv_import
 
-        undo_audioscrobbler_tsv_import(self.process_log, dryrun)
+class AudioScrobblerTSVImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "AudioScrobbler TSV Import"
 
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'audioscrobbler-uploads/{uuid}.{extension}'
 
-class LastFmImport(TimeStampedModel):
-    user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
-    uuid = models.UUIDField(editable=False, default=uuid4)
-    processing_started = models.DateTimeField(**BNULL)
-    processed_finished = models.DateTimeField(**BNULL)
-    process_log = models.TextField(**BNULL)
-    process_count = models.IntegerField(**BNULL)
+    tsv_file = models.FileField(upload_to=get_path, **BNULL)
+
+    def process(self, force=False):
+        from scrobbles.tsv import process_audioscrobbler_tsv_file
+
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        self.mark_started()
+
+        tz = None
+        if self.user:
+            tz = self.user.profile.tzinfo
+        scrobbles = process_audioscrobbler_tsv_file(
+            self.tsv_file.path, self.user.id, user_tz=tz
+        )
+        self.record_log(scrobbles)
+        self.mark_finished()
 
-    def __str__(self):
-        return f"LastFM Import: {self.uuid}"
+
+class LastFmImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "Last.FM Import"
 
     def process(self, import_all=False):
         """Import scrobbles found on LastFM"""
@@ -111,36 +184,12 @@ class LastFmImport(TimeStampedModel):
         if last_import:
             last_processed = last_import.processed_finished
 
-        self.processing_started = timezone.now()
-        self.save(update_fields=['processing_started'])
+        self.mark_started()
 
         scrobbles = lastfm.import_from_lastfm(last_processed)
-        self.process_log = ""
-        if scrobbles:
-            for count, scrobble in enumerate(scrobbles):
-                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
-                log_line = f"{scrobble_str}"
-                if count > 0:
-                    log_line = "\n" + log_line
-                self.process_log += log_line
-            self.process_count = len(scrobbles)
-        else:
-            self.process_count = 0
 
-        self.processed_finished = timezone.now()
-        self.save(
-            update_fields=[
-                'processed_finished',
-                'process_count',
-                'process_log',
-            ]
-        )
-
-    def undo(self, dryrun=False):
-        """Undo import of scrobbles from LastFM"""
-        LastFM.undo_lastfm_import(self.process_log, dryrun)
-        self.processed_finished = None
-        self.save(update_fields=['processed_finished'])
+        self.record_log(scrobbles)
+        self.mark_finished()
 
 
 class ChartRecord(TimeStampedModel):
@@ -231,19 +280,29 @@ class Scrobble(TimeStampedModel):
     sport_event = models.ForeignKey(
         SportEvent, on_delete=models.DO_NOTHING, **BNULL
     )
+    book = models.ForeignKey(Book, on_delete=models.DO_NOTHING, **BNULL)
     user = models.ForeignKey(
         User, blank=True, null=True, on_delete=models.DO_NOTHING
     )
+
+    # Time keeping
     timestamp = models.DateTimeField(**BNULL)
     playback_position_ticks = models.PositiveBigIntegerField(**BNULL)
     playback_position = models.CharField(max_length=8, **BNULL)
+
+    # Status indicators
     is_paused = models.BooleanField(default=False)
     played_to_completion = models.BooleanField(default=False)
+    in_progress = models.BooleanField(default=True)
+
+    # Metadata
     source = models.CharField(max_length=255, **BNULL)
     source_id = models.TextField(**BNULL)
-    in_progress = models.BooleanField(default=True)
     scrobble_log = models.TextField(**BNULL)
 
+    # Fields for keeping track of reads between scrobbles
+    book_pages_read = models.IntegerField(**BNULL)
+
     def save(self, *args, **kwargs):
         if not self.uuid:
             self.uuid = uuid4()
@@ -272,7 +331,10 @@ class Scrobble(TimeStampedModel):
 
     @property
     def percent_played(self) -> int:
-        if not self.media_obj.run_time_ticks:
+        if not self.media_obj:
+            return 0
+
+        if self.media_obj and not self.media_obj.run_time_ticks:
             return 100
 
         if not self.playback_position_ticks and self.played_to_completion:
@@ -309,6 +371,8 @@ class Scrobble(TimeStampedModel):
             media_obj = self.podcast_episode
         if self.sport_event:
             media_obj = self.sport_event
+        if self.book:
+            media_obj = self.book
         return media_obj
 
     def __str__(self):
@@ -332,6 +396,9 @@ class Scrobble(TimeStampedModel):
         if media.__class__.__name__ == 'SportEvent':
             media_query = models.Q(sport_event=media)
             scrobble_data['sport_event_id'] = media.id
+        if media.__class__.__name__ == 'Book':
+            media_query = models.Q(book=media)
+            scrobble_data['book_id'] = media.id
 
         scrobble = (
             cls.objects.filter(

vrobbler/apps/scrobbles/tasks.py

@@ -1,7 +1,11 @@
 import logging
 from celery import shared_task
 
-from scrobbles.models import AudioScrobblerTSVImport, LastFmImport
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    KoReaderImport,
+    LastFmImport,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -22,3 +26,12 @@ def process_tsv_import(import_id):
         logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}")
 
     tsv_import.process()
+
+
+@shared_task
+def process_koreader_import(import_id):
+    koreader_import = KoReaderImport.objects.filter(id=import_id).first()
+    if not koreader_import:
+        logger.warn(f"KOReaderImport not found with id {import_id}")
+
+    koreader_import.process()

vrobbler/apps/scrobbles/tsv.py

@@ -78,20 +78,3 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             extra={'created_scrobbles': created},
         )
         return created
-
-
-def undo_audioscrobbler_tsv_import(process_log, dryrun=False):
-    """Accepts the log from a TSV import and removes the scrobbles"""
-    if not process_log:
-        logger.warning("No lines in process log found to undo")
-        return
-
-    for line in process_log.split('\n'):
-        scrobble_id = line.split("\t")[0]
-        scrobble = Scrobble.objects.filter(id=scrobble_id).first()
-        if not scrobble:
-            logger.warning(f"Could not find scrobble {scrobble_id} to undo")
-            continue
-        logger.info(f"Removing scrobble {scrobble_id}")
-        if not dryrun:
-            scrobble.delete()

vrobbler/settings.py

@@ -97,6 +97,7 @@ INSTALLED_APPS = [
     "music",
     "podcasts",
     "sports",
+    "books",
     "mathfilters",
     "rest_framework",
     "allauth",