Restructure into full project layout
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Archidekt collection importer.
|
||||
|
||||
Supports two export formats:
|
||||
- CSV (Collection -> Export -> CSV)
|
||||
- JSON (Collection -> Export -> JSON)
|
||||
|
||||
CSV columns (current Archidekt format, as of 2024):
|
||||
Quantity, Foil Quantity, Card Name, Set Code, Collector Number, ...
|
||||
|
||||
JSON format (array of card objects):
|
||||
[{"quantity": 1, "foilQuantity": 0, "card": {"name": "...", "set": {"code": "..."}, "collectorNumber": "..."}}]
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from typing import Union
|
||||
|
||||
import chardet
|
||||
|
||||
from app.services.imports.models import RawCardRow
|
||||
|
||||
SOURCE = "archidekt"
|
||||
|
||||
# CSV column name aliases — Archidekt has changed these over time
|
||||
_NAME_COLS = {"card name", "name", "cardname"}
|
||||
_QTY_COLS = {"quantity", "qty", "count", "amount"}
|
||||
_FOIL_COLS = {"foil quantity", "foil qty", "foilqty", "foilcount", "foils"}
|
||||
_SET_COLS = {"set code", "set", "setcode", "edition"}
|
||||
_COLLECTOR_COLS = {"collector number", "collector #", "collectornumber", "number"}
|
||||
|
||||
|
||||
def parse(raw_bytes: bytes) -> list[RawCardRow]:
|
||||
"""
|
||||
Auto-detect CSV vs JSON and dispatch to the appropriate parser.
|
||||
Raises ValueError with a human-readable message on unrecognised format.
|
||||
"""
|
||||
# Detect encoding
|
||||
detected = chardet.detect(raw_bytes)
|
||||
encoding = detected.get("encoding") or "utf-8"
|
||||
text = raw_bytes.decode(encoding, errors="replace").strip()
|
||||
|
||||
if text.startswith("[") or text.startswith("{"):
|
||||
return _parse_json(text)
|
||||
else:
|
||||
return _parse_csv(text)
|
||||
|
||||
|
||||
def _parse_csv(text: str) -> list[RawCardRow]:
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("Archidekt CSV appears to be empty or has no header row")
|
||||
|
||||
# Normalise header names
|
||||
headers = {h.strip().lower(): h for h in reader.fieldnames if h}
|
||||
|
||||
name_col = _find_col(headers, _NAME_COLS, "Card Name")
|
||||
qty_col = _find_col(headers, _QTY_COLS, "Quantity")
|
||||
foil_col = _find_col(headers, _FOIL_COLS, None, required=False)
|
||||
set_col = _find_col(headers, _SET_COLS, None, required=False)
|
||||
collect_col = _find_col(headers, _COLLECTOR_COLS, None, required=False)
|
||||
|
||||
rows: list[RawCardRow] = []
|
||||
for i, row in enumerate(reader, start=2): # start=2 because row 1 is header
|
||||
name = row.get(name_col, "").strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
try:
|
||||
qty = int(float(row.get(qty_col, "1") or "1"))
|
||||
except (ValueError, TypeError):
|
||||
qty = 1
|
||||
|
||||
try:
|
||||
foil_qty = int(float(row.get(foil_col, "0") or "0")) if foil_col else 0
|
||||
except (ValueError, TypeError):
|
||||
foil_qty = 0
|
||||
|
||||
rows.append(RawCardRow(
|
||||
card_name=name,
|
||||
set_code=(row.get(set_col, "") or "").strip().lower() if set_col else "",
|
||||
collector_number=(row.get(collect_col, "") or "").strip() if collect_col else "",
|
||||
quantity=qty,
|
||||
foil_quantity=foil_qty,
|
||||
import_source=SOURCE,
|
||||
raw=dict(row),
|
||||
))
|
||||
|
||||
if not rows:
|
||||
raise ValueError("Archidekt CSV contained no card rows")
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_json(text: str) -> list[RawCardRow]:
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Archidekt JSON is malformed: {e}")
|
||||
|
||||
# Handle both bare array and {"cards": [...]} wrapper
|
||||
if isinstance(data, dict):
|
||||
data = data.get("cards") or data.get("collection") or []
|
||||
if not isinstance(data, list):
|
||||
raise ValueError("Archidekt JSON must be an array of card objects")
|
||||
|
||||
rows: list[RawCardRow] = []
|
||||
for item in data:
|
||||
card = item.get("card") or item # nested or flat
|
||||
name = (
|
||||
card.get("name")
|
||||
or card.get("cardName")
|
||||
or card.get("card_name")
|
||||
or ""
|
||||
).strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
qty = int(item.get("quantity") or item.get("qty") or 1)
|
||||
foil_qty = int(item.get("foilQuantity") or item.get("foil_quantity") or 0)
|
||||
|
||||
set_obj = card.get("set") or {}
|
||||
set_code = (
|
||||
set_obj.get("code") if isinstance(set_obj, dict) else str(set_obj)
|
||||
).lower().strip()
|
||||
|
||||
collector = str(card.get("collectorNumber") or card.get("collector_number") or "").strip()
|
||||
|
||||
rows.append(RawCardRow(
|
||||
card_name=name,
|
||||
set_code=set_code,
|
||||
collector_number=collector,
|
||||
quantity=qty,
|
||||
foil_quantity=foil_qty,
|
||||
import_source=SOURCE,
|
||||
raw=item,
|
||||
))
|
||||
|
||||
if not rows:
|
||||
raise ValueError("Archidekt JSON contained no card entries")
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _find_col(
|
||||
headers: dict[str, str],
|
||||
aliases: set[str],
|
||||
fallback: str | None,
|
||||
required: bool = True,
|
||||
) -> str | None:
|
||||
for alias in aliases:
|
||||
if alias in headers:
|
||||
return headers[alias]
|
||||
if fallback and fallback.lower() in headers:
|
||||
return headers[fallback.lower()]
|
||||
if required:
|
||||
raise ValueError(
|
||||
f"Could not find required column. Expected one of: {sorted(aliases)}"
|
||||
)
|
||||
return None
|
||||
@@ -0,0 +1,63 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app.models.collection import CollectionCard
|
||||
from app.services.imports.models import RawCardRow
|
||||
from app.services import scryfall
|
||||
|
||||
|
||||
async def enrich_and_upsert(rows: list[RawCardRow], user_id: int, db: AsyncSession) -> int:
|
||||
# Try set+collector first for exact matches
|
||||
enriched = {}
|
||||
name_lookup = []
|
||||
|
||||
for row in rows:
|
||||
if row.set_code and row.collector_number:
|
||||
card = await scryfall.get_card_by_set_and_collector(row.set_code, row.collector_number)
|
||||
if card:
|
||||
enriched[row.card_name.lower()] = card
|
||||
continue
|
||||
name_lookup.append(row.card_name)
|
||||
|
||||
# Batch enrich remaining by name
|
||||
if name_lookup:
|
||||
name_map = await scryfall.batch_enrich_by_name(name_lookup)
|
||||
enriched.update(name_map)
|
||||
|
||||
count = 0
|
||||
for row in rows:
|
||||
sf_data = enriched.get(row.card_name.lower())
|
||||
if not sf_data:
|
||||
continue
|
||||
|
||||
scryfall_id = sf_data["id"]
|
||||
|
||||
# Upsert — add quantities if card already exists
|
||||
result = await db.execute(
|
||||
select(CollectionCard).where(
|
||||
CollectionCard.owner_id == user_id,
|
||||
CollectionCard.scryfall_id == scryfall_id,
|
||||
)
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
existing.quantity += row.quantity
|
||||
existing.foil_quantity += row.foil_quantity
|
||||
existing.scryfall_data = sf_data
|
||||
else:
|
||||
db.add(CollectionCard(
|
||||
owner_id=user_id,
|
||||
card_name=sf_data.get("name", row.card_name),
|
||||
set_code=sf_data.get("set", row.set_code),
|
||||
collector_number=sf_data.get("collector_number", row.collector_number),
|
||||
quantity=row.quantity,
|
||||
foil_quantity=row.foil_quantity,
|
||||
scryfall_id=scryfall_id,
|
||||
scryfall_data=sf_data,
|
||||
))
|
||||
count += 1
|
||||
|
||||
await db.commit()
|
||||
return count
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
Manabox collection importer.
|
||||
|
||||
Manabox exports a single CSV format:
|
||||
Name, Set code, Collector number, Foil, Quantity, ...
|
||||
|
||||
The "Foil" column is a boolean ("Yes"/"No" or "1"/"0") applied per-row.
|
||||
Manabox creates separate rows for foil and non-foil copies of the same card,
|
||||
so we merge them into a single RawCardRow (quantity + foil_quantity).
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from collections import defaultdict
|
||||
|
||||
import chardet
|
||||
|
||||
from app.services.imports.models import RawCardRow
|
||||
|
||||
SOURCE = "manabox"
|
||||
|
||||
_NAME_COLS = {"name", "card name", "cardname", "card_name"}
|
||||
_SET_COLS = {"set code", "set", "setcode", "set_code", "edition code"}
|
||||
_COLLECTOR_COLS = {"collector number", "collector #", "collectornumber", "number"}
|
||||
_FOIL_COLS = {"foil", "is foil", "isfoil"}
|
||||
_QTY_COLS = {"quantity", "qty", "count", "amount"}
|
||||
|
||||
|
||||
def parse(raw_bytes: bytes) -> list[RawCardRow]:
|
||||
detected = chardet.detect(raw_bytes)
|
||||
encoding = detected.get("encoding") or "utf-8"
|
||||
text = raw_bytes.decode(encoding, errors="replace").strip()
|
||||
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("Manabox CSV appears to be empty or has no header row")
|
||||
|
||||
headers = {h.strip().lower(): h for h in reader.fieldnames if h}
|
||||
|
||||
name_col = _find_col(headers, _NAME_COLS, "Name")
|
||||
set_col = _find_col(headers, _SET_COLS, None, required=False)
|
||||
collect_col = _find_col(headers, _COLLECTOR_COLS, None, required=False)
|
||||
foil_col = _find_col(headers, _FOIL_COLS, None, required=False)
|
||||
qty_col = _find_col(headers, _QTY_COLS, "Quantity")
|
||||
|
||||
# Key: (card_name, set_code, collector_number) → merged row
|
||||
merged: dict[tuple, RawCardRow] = defaultdict(lambda: RawCardRow(import_source=SOURCE))
|
||||
|
||||
for row in reader:
|
||||
name = row.get(name_col, "").strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
set_code = (row.get(set_col, "") or "").strip().lower() if set_col else ""
|
||||
collector = (row.get(collect_col, "") or "").strip() if collect_col else ""
|
||||
key = (name.lower(), set_code, collector)
|
||||
|
||||
try:
|
||||
qty = int(float(row.get(qty_col, "1") or "1"))
|
||||
except (ValueError, TypeError):
|
||||
qty = 1
|
||||
|
||||
is_foil = _parse_bool(row.get(foil_col, "")) if foil_col else False
|
||||
|
||||
entry = merged[key]
|
||||
entry.card_name = name
|
||||
entry.set_code = set_code
|
||||
entry.collector_number = collector
|
||||
|
||||
if is_foil:
|
||||
entry.foil_quantity += qty
|
||||
else:
|
||||
entry.quantity += qty
|
||||
|
||||
if not merged:
|
||||
raise ValueError("Manabox CSV contained no card rows")
|
||||
|
||||
return list(merged.values())
|
||||
|
||||
|
||||
def _parse_bool(value: str) -> bool:
|
||||
return str(value).strip().lower() in {"yes", "true", "1", "y"}
|
||||
|
||||
|
||||
def _find_col(
|
||||
headers: dict[str, str],
|
||||
aliases: set[str],
|
||||
fallback: str | None,
|
||||
required: bool = True,
|
||||
) -> str | None:
|
||||
for alias in aliases:
|
||||
if alias in headers:
|
||||
return headers[alias]
|
||||
if fallback and fallback.lower() in headers:
|
||||
return headers[fallback.lower()]
|
||||
if required:
|
||||
raise ValueError(
|
||||
f"Could not find required column. Expected one of: {sorted(aliases)}"
|
||||
)
|
||||
return None
|
||||
@@ -0,0 +1,12 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class RawCardRow:
|
||||
card_name: str = ""
|
||||
set_code: str = ""
|
||||
collector_number: str = ""
|
||||
quantity: int = 0
|
||||
foil_quantity: int = 0
|
||||
import_source: str = ""
|
||||
raw: dict = field(default_factory=dict)
|
||||
Reference in New Issue
Block a user