Big fix to aggregation

vrobbler/apps/music/aggregators.py

@@ -78,13 +78,13 @@ def week_of_scrobbles(
         media_filter = Q(video__video_type=Video.VideoType.TV_EPISODE)
 
     for day in range(6, -1, -1):
-        start = start - timedelta(days=day)
-        end = datetime.combine(start, datetime.max.time(), now.tzinfo)
-        day_of_week = start.strftime('%A')
+        start_day = start - timedelta(days=day)
+        end = datetime.combine(start_day, datetime.max.time(), now.tzinfo)
+        day_of_week = start_day.strftime('%A')
 
         scrobble_day_dict[day_of_week] = base_qs.filter(
             media_filter,
-            timestamp__gte=start,
+            timestamp__gte=start_day,
             timestamp__lte=end,
             played_to_completion=True,
         ).count()
@@ -93,7 +93,7 @@ def week_of_scrobbles(
 
 
 def top_tracks(
-    user: "User", filter: str = "today", limit: int = 15
+    user: "User", filter: str = "today", limit: int = 30
 ) -> List["Track"]:
 
     now = timezone.now()
@@ -109,7 +109,9 @@ def top_tracks(
     starting_day_of_current_month = now.date().replace(day=1)
     starting_day_of_current_year = now.date().replace(month=1, day=1)
 
-    time_filter = Q(scrobble__timestamp__gte=start_of_today)
+    time_filter = Q()
+    if filter == "today":
+        time_filter = Q(scrobble__timestamp__gte=start_of_today)
     if filter == "week":
         time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_week)
     if filter == "month":
@@ -119,7 +121,13 @@ def top_tracks(
 
     return (
         Track.objects.filter(time_filter)
-        .annotate(num_scrobbles=Count("scrobble", distinct=True))
+        .annotate(
+            num_scrobbles=Count(
+                "scrobble",
+                filter=Q(scrobble__played_to_completion=True),
+                distinct=True,
+            )
+        )
         .order_by("-num_scrobbles")[:limit]
     )
 
@@ -143,8 +151,14 @@ def top_artists(
 
     return (
         Artist.objects.filter(time_filter)
-        .annotate(num_scrobbles=Count("track__scrobble", distinct=True))
-        .order_by("-num_scrobbles")[:limit]
+        .annotate(
+            num_scrobbles=Count(
+                "track__scrobble",
+                filter=Q(track__scrobble__played_to_completion=True),
+                distinct=True,
+            )
+        )
+        .order_by("-num_scrobbles")
     )
 
 

vrobbler/apps/music/utils.py

@@ -1,6 +1,8 @@
 import logging
 import re
 
+from musicbrainzngs.caa import musicbrainz
+
 from scrobbles.musicbrainz import (
     lookup_album_dict_from_mb,
     lookup_artist_from_mb,
@@ -21,20 +23,20 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist:
         name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
     if 'featuring' in name.lower():
         name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+    if '&' in name.lower():
+        name = re.split("&", name, flags=re.IGNORECASE)[0].strip()
 
     artist_dict = lookup_artist_from_mb(name)
     mbid = mbid or artist_dict['id']
 
     logger.debug(f'Looking up artist {name} and mbid: {mbid}')
-    artist, created = Artist.objects.get_or_create(
-        name=name, musicbrainz_id=mbid
-    )
-
-    logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
-    # Clean up bad names in our DB with MB names
-    # if artist.name != artist_dict["name"]:
-    #    artist.name = artist_dict["name"]
-    #    artist.save(update_fields=["name"])
+    artist = Artist.objects.filter(musicbrainz_id=mbid).first()
+    if not artist:
+        artist = Artist.objects.create(name=name, musicbrainz_id=mbid)
+        logger.debug(
+            f"Created artist {artist.name} ({artist.musicbrainz_id}) "
+        )
+        # TODO Enrich artist with MB data
 
     return artist
 
@@ -78,27 +80,22 @@ def get_or_create_album(name: str, artist: Artist, mbid: str = None) -> Album:
 
 def get_or_create_track(
     title: str,
-    mbid: str,
     artist: Artist,
     album: Album,
+    mbid: str = None,
     run_time=None,
     run_time_ticks=None,
 ) -> Track:
     track = None
-    if mbid:
-        track = Track.objects.filter(
-            musicbrainz_id=mbid,
-        ).first()
-    if not track:
-        track = Track.objects.filter(
-            title=title, artist=artist, album=album
-        ).first()
-
     if not mbid:
         mbid = lookup_track_from_mb(
-            title, artist.musicbrainz_id, album.musicbrainz_id
+            title,
+            artist.musicbrainz_id,
+            album.musicbrainz_id,
         )['id']
 
+    track = Track.objects.filter(musicbrainz_id=mbid).first()
+
     if not track:
         track = Track.objects.create(
             title=title,

vrobbler/apps/scrobbles/admin.py

@@ -47,6 +47,7 @@ class ChartRecordAdmin(admin.ModelAdmin):
     list_display = (
         "user",
         "rank",
+        "count",
         "year",
         "week",
         "month",

vrobbler/apps/scrobbles/context_processors.py

@@ -0,0 +1,19 @@
+import pytz
+from django.utils import timezone
+from scrobbles.models import Scrobble
+
+
+def now_playing(request):
+    user = request.user
+    now = timezone.now()
+    if user.is_authenticated:
+        if user.profile:
+            timezone.activate(pytz.timezone(user.profile.timezone))
+            now = timezone.localtime(timezone.now())
+        return {
+            'now_playing_list': Scrobble.objects.filter(
+                in_progress=True,
+                is_paused=False,
+                user=user,
+            )
+        }

vrobbler/apps/scrobbles/models.py

@@ -2,18 +2,19 @@ import calendar
 import logging
 from uuid import uuid4
 
+from books.models import Book
 from django.contrib.auth import get_user_model
 from django.db import models
+from django.urls import reverse
 from django.utils import timezone
 from django_extensions.db.models import TimeStampedModel
 from music.models import Artist, Track
-from books.models import Book
 from podcasts.models import Episode
-from profiles.utils import now_user_timezone
 from scrobbles.lastfm import LastFM
 from scrobbles.utils import check_scrobble_for_finish
 from sports.models import SportEvent
 from videos.models import Series, Video
+
 from vrobbler.apps.scrobbles.stats import build_charts
 
 logger = logging.getLogger(__name__)
@@ -35,6 +36,24 @@ class BaseFileImportMixin(TimeStampedModel):
     def __str__(self):
         return f"Scrobble import {self.id}"
 
+    @property
+    def human_start(self):
+        start = "Unknown"
+        if self.processing_started:
+            start = self.processing_started.strftime('%B %d, %Y at %H:%M')
+        return start
+
+    @property
+    def import_type(self) -> str:
+        class_name = self.__class__.__name__
+        if class_name == 'AudioscrobblerTSVImport':
+            return "Audioscrobbler"
+        if class_name == 'KoReaderImport':
+            return "KoReader"
+        if self.__class__.__name__ == 'LastFMImport':
+            return "LastFM"
+        return "Generic"
+
     def process(self, force=False):
         logger.warning("Process not implemented")
 
@@ -99,6 +118,14 @@ class KoReaderImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "KOReader Import"
 
+    def __str__(self):
+        return f"KoReader import on {self.human_start}"
+
+    def get_absolute_url(self):
+        return reverse(
+            'scrobbles:koreader-import-detail', kwargs={'slug': self.uuid}
+        )
+
     def get_path(instance, filename):
         extension = filename.split('.')[-1]
         uuid = instance.uuid
@@ -127,6 +154,14 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "AudioScrobbler TSV Import"
 
+    def __str__(self):
+        return f"Audioscrobbler import on {self.human_start}"
+
+    def get_absolute_url(self):
+        return reverse(
+            'scrobbles:tsv-import-detail', kwargs={'slug': self.uuid}
+        )
+
     def get_path(instance, filename):
         extension = filename.split('.')[-1]
         uuid = instance.uuid
@@ -159,6 +194,14 @@ class LastFmImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "Last.FM Import"
 
+    def __str__(self):
+        return f"LastFM import on {self.human_start}"
+
+    def get_absolute_url(self):
+        return reverse(
+            'scrobbles:lastfm-import-detail', kwargs={'slug': self.uuid}
+        )
+
     def process(self, import_all=False):
         """Import scrobbles found on LastFM"""
         if self.processed_finished:

vrobbler/apps/scrobbles/scrobblers.py

@@ -89,10 +89,6 @@ def mopidy_scrobble_track(
         "mopidy_status": data_dict.get("status"),
     }
 
-    # Jellyfin MB ids suck, so always overwrite with Mopidy if they're offering
-    track.musicbrainz_id = data_dict.get("musicbrainz_track_id")
-    track.save()
-
     scrobble = Scrobble.create_or_update(track, user_id, mopidy_data)
 
     return scrobble
@@ -130,7 +126,7 @@ def jellyfin_scrobble_track(
     )
 
     # Jellyfin has some race conditions with it's webhooks, these hacks fix some of them
-    if not data_dict.get("PlaybackPositionTicks") or null_position_on_progress:
+    if null_position_on_progress:
         logger.error("No playback position tick from Jellyfin, aborting")
         return
 
@@ -152,7 +148,6 @@ def jellyfin_scrobble_track(
     )
     track = get_or_create_track(
         title=data_dict.get("Name"),
-        mbid=data_dict.get(JELLYFIN_POST_KEYS["TRACK_MB_ID"]),
         artist=artist,
         album=album,
         run_time_ticks=run_time_ticks,

vrobbler/apps/scrobbles/stats.py

@@ -137,4 +137,6 @@ def build_charts(
         if model_str == 'Artist':
             chart_record['artist'] = result
         chart_records.append(ChartRecord(**chart_record))
-    ChartRecord.objects.bulk_create(chart_records, ignore_conflicts=True)
+    ChartRecord.objects.bulk_create(
+        chart_records, ignore_conflicts=True, batch_size=500
+    )

vrobbler/apps/scrobbles/urls.py

@@ -42,6 +42,26 @@ urlpatterns = [
         name='mopidy-webhook',
     ),
     path('export/', views.export, name='export'),
+    path(
+        'imports/',
+        views.ScrobbleImportListView.as_view(),
+        name='import-detail',
+    ),
+    path(
+        'imports/tsv/<slug:slug>/',
+        views.ScrobbleTSVImportDetailView.as_view(),
+        name='tsv-import-detail',
+    ),
+    path(
+        'imports/lastfm/<slug:slug>/',
+        views.ScrobbleLastFMImportDetailView.as_view(),
+        name='lastfm-import-detail',
+    ),
+    path(
+        'imports/koreader/<slug:slug>/',
+        views.ScrobbleKoReaderImportDetailView.as_view(),
+        name='koreader-import-detail',
+    ),
     path(
         'charts/',
         views.ChartRecordView.as_view(),

vrobbler/apps/scrobbles/views.py

@@ -1,7 +1,8 @@
 import calendar
 import json
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
+from django.db.models.query import QuerySet
 
 import pytz
 from django.conf import settings
@@ -13,7 +14,7 @@ from django.http import FileResponse, HttpResponseRedirect, JsonResponse
 from django.urls import reverse, reverse_lazy
 from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
-from django.views.generic import FormView, TemplateView
+from django.views.generic import DetailView, FormView, TemplateView
 from django.views.generic.edit import CreateView
 from django.views.generic.list import ListView
 from music.aggregators import (
@@ -70,17 +71,7 @@ class RecentScrobbleList(ListView):
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
         user = self.request.user
-        now = timezone.now()
         if user.is_authenticated:
-            if user.profile:
-                timezone.activate(pytz.timezone(user.profile.timezone))
-                now = timezone.localtime(timezone.now())
-            data['now_playing_list'] = Scrobble.objects.filter(
-                in_progress=True,
-                is_paused=False,
-                timestamp__lte=now,
-                user=user,
-            )
 
             completed_for_user = Scrobble.objects.filter(
                 played_to_completion=True, user=user
@@ -97,19 +88,16 @@ class RecentScrobbleList(ListView):
                 sport_event__isnull=False
             ).order_by('-timestamp')[:15]
 
-            # data['top_daily_tracks'] = top_tracks()
-            data['top_weekly_tracks'] = top_tracks(user, filter='week')
-            data['top_monthly_tracks'] = top_tracks(user, filter='month')
-
-            # data['top_daily_artists'] = top_artists()
-            data['top_weekly_artists'] = top_artists(user, filter='week')
-            data['top_monthly_artists'] = top_artists(user, filter='month')
-
         data["weekly_data"] = week_of_scrobbles(user=user)
 
         data['counts'] = scrobble_counts(user)
         data['imdb_form'] = ScrobbleForm
         data['export_form'] = ExportScrobbleForm
+        data['active_imports'] = AudioScrobblerTSVImport.objects.filter(
+            processing_started__isnull=False,
+            processed_finished__isnull=True,
+            user=self.request.user,
+        )
         return data
 
     def get_queryset(self):
@@ -118,6 +106,57 @@ class RecentScrobbleList(ListView):
         ).order_by('-timestamp')[:15]
 
 
+class ScrobbleImportListView(TemplateView):
+    template_name = "scrobbles/import_list.html"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        context_data['object_list'] = []
+
+        context_data["tsv_imports"] = AudioScrobblerTSVImport.objects.filter(
+            user=self.request.user,
+        ).order_by('-processing_started')
+        context_data["koreader_imports"] = KoReaderImport.objects.filter(
+            user=self.request.user,
+        ).order_by('-processing_started')
+        context_data["lastfm_imports"] = LastFmImport.objects.filter(
+            user=self.request.user,
+        ).order_by('-processing_started')
+        return context_data
+
+
+class BaseScrobbleImportDetailView(DetailView):
+    slug_field = 'uuid'
+    template_name = "scrobbles/import_detail.html"
+
+    def get_queryset(self):
+        return super().get_queryset().filter(user=self.request.user)
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        title = "Generic Scrobble Import"
+        if self.model == KoReaderImport:
+            title = "KoReader Import"
+        if self.model == AudioScrobblerTSVImport:
+            title = "Audioscrobbler TSV Import"
+        if self.model == LastFmImport:
+            title = "LastFM Import"
+        context_data['title'] = title
+        return context_data
+
+
+class ScrobbleKoReaderImportDetailView(BaseScrobbleImportDetailView):
+    model = KoReaderImport
+
+
+class ScrobbleTSVImportDetailView(BaseScrobbleImportDetailView):
+    model = AudioScrobblerTSVImport
+
+
+class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView):
+    model = LastFmImport
+
+
 class ManualScrobbleView(FormView):
     form_class = ScrobbleForm
     template_name = 'scrobbles/manual_form.html'
@@ -366,44 +405,116 @@ def export(request):
 class ChartRecordView(TemplateView):
     template_name = 'scrobbles/chart_index.html'
 
+    @staticmethod
+    def get_media_filter(media_type: str = "Track"):
+        media_filter = Q()
+        if media_type == 'Track':
+            media_filter = Q(track__isnull=False)
+        if media_type == 'Artist':
+            media_filter = Q(artist__isnull=False)
+        if media_type == 'Series':
+            media_filter = Q(series__isnull=False)
+        if media_type == 'Video':
+            media_filter = Q(video__isnull=False)
+        return media_filter
+
+    def get_chart_records(self, media_type: str = "Track", **kwargs):
+        media_filter = self.get_media_filter(media_type)
+        charts = ChartRecord.objects.filter(
+            media_filter, user=self.request.user, **kwargs
+        ).order_by("rank")
+
+        if charts.count() == 0:
+            ChartRecord.build(
+                user=self.request.user, model_str=media_type, **kwargs
+            )
+            charts = ChartRecord.objects.filter(
+                media_filter, user=self.request.user, **kwargs
+            ).order_by("rank")
+        return charts
+
+    def get_chart(
+        self, period: str = "all_time", limit=15, media: str = "Track"
+    ) -> QuerySet:
+        chart = QuerySet()
+        now = timezone.now()
+        if period == "all_time":
+            chart = self.get_chart_records(media_type=media)
+        if period == "today":
+            chart = self.get_chart_records(
+                media_type=media,
+                day=now.day,
+                month=now.month,
+                year=now.year,
+            )
+        if period == "week":
+            chart = self.get_chart_records(
+                media_type=media,
+                year=now.year,
+                week=now.isocalendar()[1],
+            )
+        if period == "month":
+            chart = self.get_chart_records(
+                media_type=media,
+                year=now.year,
+                month=now.month,
+            )
+        if period == "year":
+            chart = self.get_chart_records(
+                media_type=media,
+                year=now.year,
+            )
+        return chart[:limit]
+
     def get_context_data(self, **kwargs):
         context_data = super().get_context_data(**kwargs)
-
         date = self.request.GET.get('date')
         media_type = self.request.GET.get('media')
+        user = self.request.user
         params = {}
+        context_data['artist_charts'] = {}
 
-        if not media_type:
-            media_type = 'Track'
-        context_data['media_type'] = media_type
-        media_filter = Q(track__isnull=False)
-        if media_type == 'Video':
-            media_filter = Q(video__isnull=False)
-        if media_type == 'Artist':
-            media_filter = Q(artist__isnull=False)
+        if not date:
+            context_data['artist_charts'] = {
+                "today": top_artists(user, filter="today")[:30],
+                "week": top_artists(user, filter="week")[:30],
+                "month": top_artists(user, filter="month")[:30],
+                "all": top_artists(user),
+            }
 
-        year = timezone.now().year
+            context_data['track_charts'] = {
+                "today": top_tracks(user, filter="today")[:30],
+                "week": top_tracks(user, filter="week")[:30],
+                "month": top_tracks(user, filter="month")[:30],
+                "all": top_tracks(user),
+            }
+            return context_data
+
+        now = timezone.now()
+        year = now.year
         params = {'year': year}
         name = f"Chart for {year}"
 
-        if not date:
-            date = timezone.now().strftime("%Y-%m-%d")
-
         date_params = date.split('-')
         year = int(date_params[0])
+        in_progress = False
         if len(date_params) == 2:
             if 'W' in date_params[1]:
                 week = int(date_params[1].strip('W"'))
                 params['week'] = week
-                r = datetime.strptime(date + '-1', "%Y-W%W-%w").strftime(
-                    'Week of %B %d, %Y'
+                start = datetime.strptime(date + "-1", "%Y-W%W-%w").replace(
+                    tzinfo=pytz.utc
                 )
-                name = f"Chart for {r}"
+                end = start + timedelta(days=6)
+                in_progress = start <= now <= end
+                as_str = start.strftime('Week of %B %d, %Y')
+                name = f"Chart for {as_str}"
             else:
                 month = int(date_params[1])
                 params['month'] = month
                 month_str = calendar.month_name[month]
                 name = f"Chart for {month_str} {year}"
+                in_progress = now.month == month and now.year == year
         if len(date_params) == 3:
             month = int(date_params[1])
             day = int(date_params[2])
@@ -411,7 +522,11 @@ class ChartRecordView(TemplateView):
             params['day'] = day
             month_str = calendar.month_name[month]
             name = f"Chart for {month_str} {day}, {year}"
+            in_progress = (
+                now.month == month and now.year == year and now.day == day
+            )
 
+        media_filter = self.get_media_filter(media_type)
         charts = ChartRecord.objects.filter(
             media_filter, user=self.request.user, **params
         ).order_by("rank")
@@ -424,6 +539,10 @@ class ChartRecordView(TemplateView):
                 media_filter, user=self.request.user, **params
             ).order_by("rank")
 
-        context_data['object_list'] = charts
+        if in_progress:
+            # TODO recalculate
+            ...
+
         context_data['name'] = name
+        context_data['in_progress'] = in_progress
         return context_data

vrobbler/settings.py

@@ -2,6 +2,7 @@ import os
 import sys
 from pathlib import Path
 
+
 import dj_database_url
 from django.utils.translation import gettext_lazy as _
 from dotenv import load_dotenv
@@ -71,6 +72,10 @@ CSRF_TRUSTED_ORIGINS = [
 X_FRAME_OPTIONS = "SAMEORIGIN"
 
 REDIS_URL = os.getenv("VROBBLER_REDIS_URL", None)
+if REDIS_URL:
+    print(f"Sending tasks to redis@{REDIS_URL.split('@')[-1]}")
+else:
+    print("Eagerly running all tasks")
 
 CELERY_TASK_ALWAYS_EAGER = os.getenv("VROBBLER_SKIP_CELERY", False)
 CELERY_BROKER_URL = REDIS_URL if REDIS_URL else "memory://localhost/"
@@ -135,6 +140,7 @@ TEMPLATES = [
                 "django.contrib.messages.context_processors.messages",
                 "videos.context_processors.video_lists",
                 "music.context_processors.music_lists",
+                "scrobbles.context_processors.now_playing",
             ],
         },
     },
@@ -150,10 +156,18 @@ DATABASES = {
         conn_max_age=600,
     ),
 }
+
 if TESTING:
     DATABASES = {
         "default": dj_database_url.config(default="sqlite:///testdb.sqlite3")
     }
+db_str = ""
+if 'sqlite' in DATABASES['default']['ENGINE']:
+    db_str = f"Connected to sqlite@{DATABASES['default']['NAME']}"
+if 'postgresql' in DATABASES['default']['ENGINE']:
+    db_str = f"Connected to postgres@{DATABASES['default']['HOST']}/{DATABASES['default']['NAME']}"
+if db_str:
+    print(db_str)
 
 
 CACHES = {

vrobbler/templates/base.html

@@ -204,25 +204,6 @@
                         {% endfor %}
                         </ul>
                         {% endif %}
-                        {% if now_playing_list and user.is_authenticated %}
-                        <ul style="padding-right:10px;">
-                            <b>Now playing</b>
-                            {% for scrobble in now_playing_list %}
-                            <div>
-                                {{scrobble.media_obj.title}}<br/>
-                                {% if scrobble.media_obj.subtitle %}<em>{{scrobble.media_obj.subtitle}}</em><br/>{% endif %}
-                                <small>{{scrobble.timestamp|naturaltime}}<br/>
-                                    from {{scrobble.source}}</small>
-                                <div class="progress-bar" style="margin-right:5px;">
-                                    <span class="progress-bar-fill" style="width: {{scrobble.percent_played}}%;"></span>
-                                </div>
-                                <a href="{% url "scrobbles:cancel" scrobble.uuid %}">Cancel</a>
-                                <a href="{% url "scrobbles:finish" scrobble.uuid %}">Finish</a>
-                            </div>
-                            <hr/>
-                            {% endfor %}
-                        </ul>
-                        {% endif %}
                         <ul class="nav flex-column">
                             <li class="nav-item">
                                 <a class="nav-link active" aria-current="page" href="/">
@@ -271,6 +252,40 @@
                         </ul>
                         {% block extra_nav %}
                         {% endblock %}
+                        <hr/>
+
+                        {% if now_playing_list and user.is_authenticated %}
+                        <ul style="padding-right:10px;">
+                            <b>Now playing</b>
+                            {% for scrobble in now_playing_list %}
+                            <div>
+                                {% if scrobble.media_obj.album.cover_image %}
+                                <td><img src="{{scrobble.track.album.cover_image.url}}" width=120 height=120
+                                        style="border:1px solid black; " /></td><br/>
+                                {% endif %}
+                                {{scrobble.media_obj.title}}<br/>
+                                {% if scrobble.media_obj.subtitle %}<em>{{scrobble.media_obj.subtitle}}</em><br/>{% endif %}
+                                <small>{{scrobble.timestamp|naturaltime}}<br/>
+                                    from {{scrobble.source}}</small>
+                                <div class="progress-bar" style="margin-right:5px;">
+                                    <span class="progress-bar-fill" style="width: {{scrobble.percent_played}}%;"></span>
+                                </div>
+                                <a href="{% url "scrobbles:cancel" scrobble.uuid %}">Cancel</a>
+                                <a href="{% url "scrobbles:finish" scrobble.uuid %}">Finish</a>
+                                <hr />
+                            </div>
+                            {% endfor %}
+                        </ul>
+                        <hr/>
+                        {% endif %}
+
+                        {% if active_imports %}
+                        {% for import in active_imports %}
+                        <ul style="padding-right:10px;">
+                            <li>Import in progress ({{import.processing_started|naturaltime}})</li>
+                        </ul>
+                        {% endfor %}
+                        {% endif %}
 
                     </div>
                 </nav>

vrobbler/templates/scrobbles/chart_index.html

@@ -1,30 +1,94 @@
 {% extends "base_list.html" %}
 
+{% block title %}{{name}}{% endblock %}
+
 {% block lists %}
-<h2>{{name}}</h2>
 <div class="row">
-    <div class="col-md">
-        <div class="table-responsive">
-            <table class="table table-striped table-sm">
-                <thead>
-                    <tr>
-                        <th scope="col">Rank</th>
-                        <th scope="col">{{media_type}}</th>
-                        <th scope="col">Scrobbles</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for chartrecord in object_list %}
-                    <tr>
-                        <td>{{chartrecord.rank}}</td>
-                        <td><a href="{{chartrecord.media_obj.get_absolute_url}}">{{chartrecord.media_obj}}</a></td>
-                        <td>{{chartrecord.count}}</td>
-                        <td></td>
-                    </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
+    <h2>Top Artists</h2>
+
+    <ul class="nav nav-tabs" id="artistTab" role="tablist">
+        {% for chart_name in artist_charts.keys %}
+        <li class="nav-item" role="presentation">
+            <button class="nav-link {% if forloop.first %}active{% endif %}" id="artist-{{chart_name}}-tab" data-bs-toggle="tab" data-bs-target="#artist-{{chart_name}}"
+                    type="button" role="tab" aria-controls="home" aria-selected="true">
+                {{chart_name}}
+            </button>
+        </li>
+        {% endfor %}
+    </ul>
+
+    <div class="tab-content" id="artistTabContent">
+        {% for chart_name, artists in artist_charts.items %}
+        <div class="tab-pane fade {% if forloop.first %}show active{% endif %}" id="artist-{{chart_name}}" role="tabpanel"
+            aria-labelledby="artist-{[chart}}-tab">
+            <div class="table-responsive">
+                <table class="table table-striped table-sm">
+                    <thead>
+                        <tr>
+                            <th scope="col">Rank</th>
+                            <th scope="col">Artist</th>
+                            <th scope="col">Scrobbles</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for artist in artists %}
+                        <tr>
+                            <td>{{artist.rank}}</td>
+                            <td><a href="{{artist.get_absolute_url}}">{{artist}}</a></td>
+                            <td>{{artist.num_scrobbles}}</td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
         </div>
+        {% endfor %}
     </div>
 </div>
+
+<div class="row">
+    <h2>Top Tracks</h2>
+
+    <ul class="nav nav-tabs" id="artistTab" role="tablist">
+        {% for chart_name in track_charts.keys %}
+        <li class="nav-item" role="presentation">
+            <button class="nav-link {% if forloop.first %}active{% endif %}" id="track-{{chart_name}}-tab" data-bs-toggle="tab" data-bs-target="#track-{{chart_name}}"
+                    type="button" role="tab" aria-controls="home" aria-selected="true">
+                {{chart_name}}
+            </button>
+        </li>
+        {% endfor %}
+    </ul>
+
+    <div class="tab-content" id="trackTabContent">
+        {% for chart_name, tracks in track_charts.items %}
+        <div class="tab-pane fade {% if forloop.first %}show active{% endif %}" id="track-{{chart_name}}" role="tabpanel"
+            aria-labelledby="track-{[chart_name}}-tab">
+            <div class="table-responsive">
+                <table class="table table-striped table-sm">
+                    <thead>
+                        <tr>
+                            <th scope="col">Rank</th>
+                            <th scope="col">Artist</th>
+                            <th scope="col">Track</th>
+                            <th scope="col">Scrobbles</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for track in tracks %}
+                        <tr>
+                            <td>{{track.rank}}</td>
+                            <td><a href="{{track.artist.get_absolute_url}}">{{track.artist}}</a></td>
+                            <td><a href="{{track.get_absolute_url}}">{{track.title}}</a></td>
+                            <td>{{track.num_scrobbles}}</td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </div>
+        {% endfor %}
+    </div>
+</div>
+
 {% endblock %}

vrobbler/templates/scrobbles/import_detail.html

@@ -0,0 +1,13 @@
+{% extends "base_detail.html" %}
+
+{% block title %}{{title}}{% endblock %}
+
+{% block details %}
+<div class="row">
+    <div class="col-md">
+        <p>Import started: {{object.processing_started}}</p>
+        <p>Import finished: {{object.processed_finished}}</p>
+        <p>Imported {{object.process_count}} scrobbles</p>
+    </div>
+</div>
+{% endblock %}

vrobbler/templates/scrobbles/import_list.html

@@ -0,0 +1,47 @@
+{% extends "base_list.html" %}
+
+{% block title %}Scrobble Imports{% endblock %}
+{% block lists %}
+<div class="row">
+    <div class="col-md">
+        <div class="table-responsive">
+            <table class="table table-striped table-sm">
+                <thead>
+                    <tr>
+                        <th scope="col">Type</th>
+                        <th scope="col">Date</th>
+                        <th scope="col">Scrobbles Imported</th>
+                        <th scope="col">Finished</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for obj in tsv_imports %}
+                    <tr>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.import_type}}</a></td>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.human_start}}</a></td>
+                        <td>{{obj.process_count}}</td>
+                        <td>{{obj.processed_finished}}</td>
+                    </tr>
+                    {% endfor %}
+                    {% for obj in lastfm_imports %}
+                    <tr>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.import_type}}</a></td>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.human_start}}</a></td>
+                        <td>{{obj.process_count}}</td>
+                        <td>{{obj.processed_finished}}</td>
+                    </tr>
+                    {% endfor %}
+                    {% for obj in koreader_imports %}
+                    <tr>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.import_type}}</a></td>
+                        <td><a href="{{obj.get_absolute_url}}">{{obj.human_start}}</a></td>
+                        <td>{{obj.process_count}}</td>
+                        <td>{{obj.processed_finished}}</td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+    </div>
+</div>
+{% endblock %}

vrobbler/templates/scrobbles/scrobble_list.html

@@ -46,260 +46,131 @@
                 This Year <b>{{counts.year}}</b> | All Time <b>{{counts.alltime}}</b></p>
         </div>
         <div class="row">
-            <div class="col-md">
-                <ul class="nav nav-tabs" id="myTab" role="tablist">
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link active" id="home-tab" data-bs-toggle="tab"
-                            data-bs-target="#artists-week" type="button" role="tab" aria-controls="home"
-                            aria-selected="true">Weekly Artists</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="artist-month-tab" data-bs-toggle="tab"
-                            data-bs-target="#artists-month" type="button" role="tab" aria-controls="home"
-                            aria-selected="true">Monthly Artists</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#tracks-week"
-                            type="button" role="tab" aria-controls="profile" aria-selected="false">Weekly
-                            Tracks</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#tracks-month"
-                            type="button" role="tab" aria-controls="profile" aria-selected="false">Monthly
-                            Tracks</button>
-                    </li>
-                </ul>
-
-                <div class="tab-content" id="myTabContent">
-                    <div class="tab-pane fade show active" id="artists-week" role="tabpanel"
-                        aria-labelledby="artists-week-tab">
-                        <h2>Top artists this week</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">#</th>
-                                        <th scope="col">Artist</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for artist in top_weekly_artists %}
-                                    <tr>
-                                        <td>{{artist.num_scrobbles}}</td>
-                                        <td>{{artist.name}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
-                    </div>
-
-                    <div class="tab-pane fade show" id="tracks-week" role="tabpanel" aria-labelledby="tracks-week-tab">
-                        <h2>Top tracks this week</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">#</th>
-                                        <th scope="col">Track</th>
-                                        <th scope="col">Artist</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for track in top_weekly_tracks %}
-                                    <tr>
-                                        <td>{{track.num_scrobbles}}</td>
-                                        <td>{{track.title}}</td>
-                                        <td>{{track.artist.name}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
-                    </div>
-
-
-                    <div class="tab-pane fade show" id="tracks-month" role="tabpanel"
-                        aria-labelledby="tracks-month-tab">
-                        <h2>Top tracks this month</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">#</th>
-                                        <th scope="col">Track</th>
-                                        <th scope="col">Artist</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for track in top_monthly_tracks %}
-                                    <tr>
-                                        <td>{{track.num_scrobbles}}</td>
-                                        <td>{{track.title}}</td>
-                                        <td>{{track.artist.name}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
-                    </div>
-
-                    <div class="tab-pane fade show " id="artists-month" role="tabpanel"
-                        aria-labelledby="artists-month-tab">
-                        <h2>Top artists this month</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">#</th>
-                                        <th scope="col">Artist</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for artist in top_monthly_artists %}
-                                    <tr>
-                                        <td>{{artist.num_scrobbles}}</td>
-                                        <td>{{artist.name}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
+            <ul class="nav nav-tabs" id="myTab" role="tablist">
+                <li class="nav-item" role="presentation">
+                    <button class="nav-link active" id="home-tab" data-bs-toggle="tab" data-bs-target="#latest-listened"
+                        type="button" role="tab" aria-controls="home" aria-selected="true">Tracks</button>
+                </li>
+                <li class="nav-item" role="presentation">
+                    <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-watched"
+                        type="button" role="tab" aria-controls="profile" aria-selected="false">Videos</button>
+                </li>
+                <li class="nav-item" role="presentation">
+                    <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-podcasted"
+                        type="button" role="tab" aria-controls="profile" aria-selected="false">Podcasts</button>
+                </li>
+                <li class="nav-item" role="presentation">
+                    <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-sports"
+                        type="button" role="tab" aria-controls="profile" aria-selected="false">Sports</button>
+                </li>
+            </ul>
+
+            <div class="tab-content" id="myTabContent2">
+                <div class="tab-pane fade show active" id="latest-listened" role="tabpanel"
+                    aria-labelledby="latest-listened-tab">
+                    <h2>Latest listened</h2>
+                    <div class="table-responsive">
+                        <table class="table table-striped table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col">Time</th>
+                                    <th scope="col">Album</th>
+                                    <th scope="col">Track</th>
+                                    <th scope="col">Artist</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for scrobble in object_list %}
+                                <tr>
+                                    <td>{{scrobble.timestamp|naturaltime}}</td>
+                                    {% if scrobble.track.album.cover_image %}
+                                    <td><img src="{{scrobble.track.album.cover_image.url}}" width=25 height=25
+                                            style="border:1px solid black;" /></td>
+                                    {% else %}
+                                    <td>{{scrobble.track.album.name}}</td>
+                                    {% endif %}
+                                    <td>{{scrobble.track.title}}</td>
+                                    <td>{{scrobble.track.artist.name}}</td>
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
                     </div>
-
                 </div>
-            </div>
-            <div class="col-md">
-                <ul class="nav nav-tabs" id="myTab" role="tablist">
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link active" id="home-tab" data-bs-toggle="tab"
-                            data-bs-target="#latest-listened" type="button" role="tab" aria-controls="home"
-                            aria-selected="true">Tracks</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-watched"
-                            type="button" role="tab" aria-controls="profile" aria-selected="false">Videos</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab"
-                            data-bs-target="#latest-podcasted" type="button" role="tab" aria-controls="profile"
-                            aria-selected="false">Podcasts</button>
-                    </li>
-                    <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-sports"
-                            type="button" role="tab" aria-controls="profile" aria-selected="false">Sports</button>
-                    </li>
-                </ul>
-
-                <div class="tab-content" id="myTabContent2">
-                    <div class="tab-pane fade show active" id="latest-listened" role="tabpanel"
-                        aria-labelledby="latest-listened-tab">
-                        <h2>Latest listened</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">Time</th>
-                                        <th scope="col">Album</th>
-                                        <th scope="col">Track</th>
-                                        <th scope="col">Artist</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for scrobble in object_list %}
-                                    <tr>
-                                        <td>{{scrobble.timestamp|naturaltime}}</td>
-                                        {% if scrobble.track.album.cover_image %}
-                                        <td><img src="{{scrobble.track.album.cover_image.url}}" width=50 height=50
-                                                style="border:1px solid black;" /></td>
-                                        {% else %}
-                                        <td>{{scrobble.track.album.name}}</td>
-                                        {% endif %}
-                                        <td>{{scrobble.track.title}}</td>
-                                        <td>{{scrobble.track.artist.name}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
-                    </div>
 
-                    <div class="tab-pane fade show" id="latest-watched" role="tabpanel"
-                        aria-labelledby="latest-watched-tab">
-                        <h2>Latest watched</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">Time</th>
-                                        <th scope="col">Title</th>
-                                        <th scope="col">Series</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for scrobble in video_scrobble_list %}
-                                    <tr>
-                                        <td>{{scrobble.timestamp|naturaltime}}</td>
-                                        <td>{% if scrobble.video.tv_series %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{% endif %} {{scrobble.video.title}}</td>
-                                        <td>{% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %}
-                                        </td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
+                <div class="tab-pane fade show" id="latest-watched" role="tabpanel"
+                    aria-labelledby="latest-watched-tab">
+                    <h2>Latest watched</h2>
+                    <div class="table-responsive">
+                        <table class="table table-striped table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col">Time</th>
+                                    <th scope="col">Title</th>
+                                    <th scope="col">Series</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for scrobble in video_scrobble_list %}
+                                <tr>
+                                    <td>{{scrobble.timestamp|naturaltime}}</td>
+                                    <td>{% if scrobble.video.tv_series%}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{%endif %} {{scrobble.video.title}}</td>
+                                    <td>{% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %}
+                                    </td>
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
                     </div>
+                </div>
 
-                    <div class="tab-pane fade show" id="latest-sports" role="tabpanel"
-                        aria-labelledby="latest-sports-tab">
-                        <h2>Latest Sports</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">Date</th>
-                                        <th scope="col">Title</th>
-                                        <th scope="col">League</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for scrobble in sport_scrobble_list %}
-                                    <tr>
-                                        <td>{{scrobble.timestamp|naturaltime}}</td>
-                                        <td>{{scrobble.sport_event.title}}</td>
-                                        <td>{{scrobble.sport_event.league.abbreviation}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
+                <div class="tab-pane fade show" id="latest-sports" role="tabpanel" aria-labelledby="latest-sports-tab">
+                    <h2>Latest Sports</h2>
+                    <div class="table-responsive">
+                        <table class="table table-striped table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col">Date</th>
+                                    <th scope="col">Title</th>
+                                    <th scope="col">League</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for scrobble in sport_scrobble_list %}
+                                <tr>
+                                    <td>{{scrobble.timestamp|naturaltime}}</td>
+                                    <td>{{scrobble.sport_event.title}}</td>
+                                    <td>{{scrobble.sport_event.league.abbreviation}}</td>
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
                     </div>
+                </div>
 
-                    <div class="tab-pane fade show" id="latest-podcasted" role="tabpanel"
-                        aria-labelledby="latest-podcasted-tab">
-                        <h2>Latest Podcasted</h2>
-                        <div class="table-responsive">
-                            <table class="table table-striped table-sm">
-                                <thead>
-                                    <tr>
-                                        <th scope="col">Date</th>
-                                        <th scope="col">Title</th>
-                                        <th scope="col">Podcast</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    {% for scrobble in podcast_scrobble_list %}
-                                    <tr>
-                                        <td>{{scrobble.timestamp|naturaltime}}</td>
-                                        <td>{{scrobble.podcast_episode.title}}</td>
-                                        <td>{{scrobble.podcast_episode.podcast}}</td>
-                                    </tr>
-                                    {% endfor %}
-                                </tbody>
-                            </table>
-                        </div>
+                <div class="tab-pane fade show" id="latest-podcasted" role="tabpanel"
+                    aria-labelledby="latest-podcasted-tab">
+                    <h2>Latest Podcasted</h2>
+                    <div class="table-responsive">
+                        <table class="table table-striped table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col">Date</th>
+                                    <th scope="col">Title</th>
+                                    <th scope="col">Podcast</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for scrobble in podcast_scrobble_list %}
+                                <tr>
+                                    <td>{{scrobble.timestamp|naturaltime}}</td>
+                                    <td>{{scrobble.podcast_episode.title}}</td>
+                                    <td>{{scrobble.podcast_episode.podcast}}</td>
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
                     </div>
-
                 </div>
             </div>
             {% endif %}