Merge branch 'develop'

vrobbler/apps/beers/untappd.py

@@ -1,6 +1,4 @@
-import json
 import logging
-import urllib
 from typing import Optional
 
 import requests
@@ -87,7 +85,9 @@ def get_ibu_from_soup(soup) -> Optional[int]:
 def get_rating_from_soup(soup) -> str:
     rating = ""
     try:
-        rating = float(soup.find(class_="num").get_text().strip("(").strip(")"))
+        rating = float(
+            soup.find(class_="num").get_text().strip("(").strip(")")
+        )
     except AttributeError:
         rating = None
     except ValueError:
@@ -130,7 +130,6 @@ def get_beer_from_untappd_id(untappd_id: str) -> dict:
         return beer_dict
 
     soup = BeautifulSoup(response.text, "html.parser")
-    beer_dict["untappd_id"] = untappd_id
     beer_dict["title"] = get_title_from_soup(soup)
     beer_dict["description"] = get_description_from_soup(soup)
     beer_dict["styles"] = get_styles_from_soup(soup)

vrobbler/apps/puzzles/models.py

@@ -1,12 +1,14 @@
 from uuid import uuid4
 
-from puzzles.sources import ipdb
+import requests
 from django.apps import apps
+from django.core.files.base import ContentFile
 from django.db import models
 from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
 from imagekit.models import ImageSpecField
 from imagekit.processors import ResizeToFit
+from puzzles.sources import ipdb
 from scrobbles.dataclasses import PuzzleLogData
 from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
 
@@ -88,17 +90,31 @@ class Puzzle(ScrobblableMixin):
 
         if not puzzle:
             puzzle_dict = ipdb.get_puzzle_from_ipdb_id(ipdb_id)
-            manufacturer_name = puzzle_dict.pop("manufacturer")
-            manufacturer, _created = PuzzleManufacturer.objects.get_or_create(
-                name=manufacturer_name
-            )
-            ipdb_dict["manufacturer_id"] = manufacturer.id
-
-            genres = puzzle_dict.pop("genres")
+            manufacturer_name = puzzle_dict.pop("manufacturer", None)
+            if manufacturer_name:
+                (
+                    manufacturer,
+                    _created,
+                ) = PuzzleManufacturer.objects.get_or_create(
+                    name=manufacturer_name
+                )
+                puzzle_dict["manufacturer_id"] = manufacturer.id
+
+            genres = puzzle_dict.pop("genres", None)
+            cover_url = puzzle_dict.pop("ipdb_image_url", None)
             puzzle = Puzzle.objects.create(**puzzle_dict)
+
             if genres:
                 puzzle.genre.add(*genres)
 
+            if cover_url:
+                r = requests.get(cover_url)
+                if r.status_code == 200:
+                    fname = f"{puzzle.title}_{puzzle.uuid}.jpg"
+                    puzzle.ipdb_image.save(
+                        fname, ContentFile(r.content), save=True
+                    )
+
         return puzzle
 
     def scrobbles(self, user_id):

vrobbler/apps/puzzles/sources/ipdb.py

@@ -1,5 +1,246 @@
-#!/usr/bin/env python3
+import logging
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+IPDB_URL = "https://www.ipdb.plus/IPDb/puzzle.php?id={id}"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def get_title_from_soup(soup) -> str:
+    title = ""
+    try:
+        title = soup.find("h1").get_text()
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return title
+
+
+def get_manufacturer_from_soup(soup) -> str:
+    manufacturer = ""
+    try:
+        manufacturer = (
+            soup.find(class_="infobox").div.contents[0].split("|")[0].strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return manufacturer
+
+
+def get_pieces_count_from_soup(soup) -> int:
+    pieces = 0
+    try:
+        pieces = int(
+            soup.find(class_="infobox")
+            .div.contents[0]
+            .split("|")[1]
+            .split(" ")[1]
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return pieces
+
+
+def get_publish_year_from_soup(soup) -> int:
+    year = 1900
+    try:
+        year = int(
+            soup.find(class_="infobox").div.contents[0].split("|")[2].strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return year
+
+
+def get_material_from_soup(soup) -> str:
+    material = ""
+    try:
+        material = (
+            soup.findAll(class_="infobox")[1]
+            .findAll("div")[1]
+            .contents[2]
+            .split("|")[-1]
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return material
+
+
+def get_cut_style_from_soup(soup) -> str:
+    cut_style = ""
+    try:
+        cut_style = (
+            soup.findAll(class_="infobox")[1]
+            .findAll("div")[1]
+            .contents[2]
+            .split("|")[-2]
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return cut_style
+
+
+def get_barcode_from_soup(soup) -> str:
+    barcode = ""
+    try:
+        barcode = (
+            soup.findAll(class_="infobox")[2]
+            .findAll("div")[0]
+            .contents[0]
+            .split("|")[-1]
+            .split(":")[-1]
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    if not barcode:
+        try:
+            barcode = (
+                soup.findAll(class_="infobox")[3]
+                .findAll("div")[0]
+                .contents[0]
+                .split("|")[-1]
+                .split(":")[-1]
+                .strip()
+            )
+        except AttributeError:
+            pass
+        except ValueError:
+            pass
+        except IndexError:
+            pass
+    return barcode
+
+
+def get_image_url_from_soup(soup) -> str:
+    url = ""
+    try:
+        url = (
+            soup.find(class_="image-container").contents[0].contents[0]["src"]
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return url
+
+
+def get_orientation_from_soup(soup) -> str:
+    orientation = ""
+    try:
+        orientation = (
+            soup.findAll(class_="infobox")[1]
+            .findAll("div")[1]
+            .contents[0]
+            .split("|")[0]
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return orientation
+
+
+def get_dimensions_from_soup(soup) -> str:
+    dimensions = ""
+    try:
+        height = (
+            soup.findAll(class_="infobox")[1]
+            .findAll("div")[1]
+            .contents[1]["data-bs-height"]
+        )
+        width = (
+            soup.findAll(class_="infobox")[1]
+            .findAll("div")[1]
+            .contents[1]["data-bs-width"]
+        )
+
+        dimensions = "x".join([height, width])
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return dimensions
+
+
+def get_rating_from_soup(soup) -> str:
+    rating = ""
+    try:
+        rating = float(
+            soup.find(class_="num").get_text().strip("(").strip(")")
+        )
+    except AttributeError:
+        rating = None
+    except ValueError:
+        rating = None
+    return rating
 
 
 def get_puzzle_from_ipdb_id(ipdb_id: str) -> dict:
-    return {}
+    puzzle_url = IPDB_URL.format(id=ipdb_id)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    response = requests.get(puzzle_url, headers=headers)
+    puzzle_dict = {"ipdb_id": ipdb_id}
+
+    if response.status_code != 200:
+        logger.warn(
+            "Bad response from untappd.com", extra={"response": response}
+        )
+        return puzzle_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    puzzle_dict["title"] = get_title_from_soup(soup)
+    puzzle_dict["orientation"] = get_orientation_from_soup(soup)
+    puzzle_dict["manufacturer"] = get_manufacturer_from_soup(soup)
+    puzzle_dict["pieces_count"] = get_pieces_count_from_soup(soup)
+    puzzle_dict["ipdb_image_url"] = get_image_url_from_soup(soup)
+    puzzle_dict["dimensions_in_inches"] = get_dimensions_from_soup(soup)
+    puzzle_dict["publish_year"] = get_publish_year_from_soup(soup)
+    puzzle_dict["material"] = get_material_from_soup(soup)
+    puzzle_dict["cut_style"] = get_cut_style_from_soup(soup)
+    puzzle_dict["barcode"] = get_barcode_from_soup(soup)
+    return puzzle_dict