moved ty python

This commit is contained in:
Felipe Martin 2022-08-20 14:43:28 +02:00
parent 339ff200ef
commit 29401d6bea
Signed by: fmartingr
GPG Key ID: 716BC147715E716F
14 changed files with 428 additions and 164 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
.env
*.pyc
*.json

3
README.md Normal file
View File

@ -0,0 +1,3 @@
# One Piece TCG Notion Importer
Simple python script to import One Piece's TCG data from onepiece-card.dev to a notion database for collection tracking.

View File

@ -1,7 +0,0 @@
package main
const (
sourcesURL = "https://onepiece-cardgame.dev/sources.json"
metaURL = "https://onepiece-cardgame.dev/meta.json"
cardsURL = "https://onepiece-cardgame.dev/cards.json"
)

View File

@ -1,25 +0,0 @@
package main
import (
"io"
"log"
"net/http"
)
func downloadURL(u string) ([]byte, error) {
log.Printf("Downloading %s\n", u)
response, err := http.Get(u)
if err != nil {
return nil, err
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
return nil, err
}
return body, nil
}

5
go.mod
View File

@ -1,5 +0,0 @@
module code.fmartingr.dev/fmartingr/onepiece-tcg-notion-importer
go 1.18
require github.com/dstotijn/go-notion v0.6.1

5
go.sum
View File

@ -1,5 +0,0 @@
github.com/dstotijn/go-notion v0.6.1 h1:gmwU/JCdLC5szMasfysDOm8UG6/3P0bTUe0+CeW2fmI=
github.com/dstotijn/go-notion v0.6.1/go.mod h1:oxd+T9Wxduj5ZN7MRiHWtyGhGZLUFsUpZHMLS4uI1Qc=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

45
main.go
View File

@ -1,45 +0,0 @@
package main
import (
"context"
"encoding/json"
"log"
"github.com/dstotijn/go-notion"
)
const (
collectionsDatabaseID = "32fc86afa91e4718b17cb79ab8794265"
)
func main() {
content, err := downloadURL(sourcesURL)
if err != nil {
panic(err)
}
log.Println(string(content))
var sources []source
if err := json.Unmarshal(content, &sources); err != nil {
panic(err)
}
ctx := context.Background()
dbSources := make(map[string]notion.Page)
client := notion.NewClient("secret_135hSjxx1xcFWcrjcS1eejjp120T86V8sIGzVeW21X4")
result, err := client.QueryDatabase(ctx, collectionsDatabaseID, &notion.DatabaseQuery{})
if err != nil {
panic(err)
}
for _, s := range result.Results {
dbSources[s.Properties.(notion.DatabasePageProperties)["SourceID"].Value().(string)] = s
}
// for _, s := range sources {
// }
}

80
onepiece_tcg/__main__.py Normal file
View File

@ -0,0 +1,80 @@
import logging
import os
import sys
from .notion import NotionClient
from .download import get_dict_from_url_response_body
from .models import Database
if __name__ == "__main__":
logging.getLogger().setLevel(logging.INFO)
try:
token = os.environ["NOTION_TOKEN"]
database_id = os.environ["NOTION_DATABASE_ID"]
card_id_property = "Card ID"
except KeyError as e:
logging.error("NOTION_TOKEN and NOTION_DATABASE_ID are required")
sys.exit(1)
database = Database()
# Download and build Meta database
logging.info("Downloading meta...")
meta = get_dict_from_url_response_body("https://onepiece-cardgame.dev/meta.json")
# for attack in meta["a"]:
# database.register_attack(attack["atk_id"], **attack)
for color in meta["c"]:
database.register_color(color["color_id"], **color)
# for source in meta["s"]:
# database.register_source(source["src_id"], **source)
for _type in meta["t"]:
database.register_type(_type["type_id"], **_type)
for card_type in meta["ct"]:
database.register_card_type(card_type["type_id"], **card_type)
for rarity in meta["r"]:
database.register_rarity(rarity["rarity_id"], **rarity)
# for key in meta['key']:
# database.register_key(key['id'], **key)
# Download and build Sources database
logging.info("Downloading sources...")
sources = get_dict_from_url_response_body(
"https://onepiece-cardgame.dev/sources.json"
)
for source in sources:
database.register_source(source["src_id"], **source)
# Donwload and build Cards database
logging.info("Downloading cards...")
cards = get_dict_from_url_response_body("https://onepiece-cardgame.dev/cards.json")
for card in cards:
database.register_card(card["cid"], **card)
# Fetch current data from Notion
notion_cli = NotionClient(token)
notion_state = dict()
try:
logging.info("Retrieving notion state...")
notion_db = notion_cli.get_database_state(database_id)
for card in notion_db["results"]:
notion_state[card[card_id_property]] = card
except Exception as e:
logging.error(e)
sys.exit(2)
logging.info("Merging databases...")
for card in database.iter_cards():
if card.card_id in notion_state:
notion_cli.update_card(notion_state[card.card_id], **{})
else:
notion_cli.create_card(database_id, **card.to_notion())

20
onepiece_tcg/download.py Normal file
View File

@ -0,0 +1,20 @@
import json
from urllib.request import urlopen, Request
from urllib.error import URLError
def get_dict_from_url_response_body(url: str) -> str:
try:
request = Request(
url,
headers={
"User-Agent": "onepiece-tcg-notion-importer",
},
)
with urlopen(request) as response:
data = response.read()
return json.loads(data)
except URLError as e:
raise e
except json.JSONDecodeError as e:
raise e

133
onepiece_tcg/models.py Normal file
View File

@ -0,0 +1,133 @@
import json
import logging
from typing import Dict, Iterator
class Card(object):
global_id: str
card_id: str
name: str
type: str
color: str
source: str
rarity: str
image_url: str
alternate_art: str
# gid: str # GlobalID: "47",
# cid: str # CardID: "OP01-047",
# n: str # Name: "Trafalgar Law",
# t: str # Type: "2",
# col: str # Color: "7",
# cs: str # Source: "5",
# tr: str # "Supernovas\/ Heart Pirates",
# a: str # AttackType: "1",
# p: str # Power: "6000",
# cp: str # null,
# l: dict # null,
# r: str # Rarity: "5",
# ar: str # null,
# iu: str # ImageURL: "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg",
# e: str # Effect: "<Blocker>\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.",
# al: dict # Alternate Art: "P1",
# intl: str # International: "0",
# srcN: str # SourceName: "Romance Dawn [OP-01]",
# srcD: dict # null
def __init__(
self,
global_id,
card_id,
name,
type,
color,
source,
rarity,
image_url,
alternate_art,
):
self.global_id = global_id
self.card_id = card_id
self.name = name
self.type = type
self.color = color
self.source = source
self.rarity = rarity
self.image_url = image_url
self.alternate_art = alternate_art
def __repr__(self):
return json.dumps(self.__dict__)
def to_notion(self):
result = {
"Name": {"title": [{"text": {"content": self.name}}]},
"Rarity": {"type": "select", "select": {"name": self.rarity}},
"Card ID": {
"type": "rich_text",
"rich_text": [
{
"type": "text",
"text": {"content": self.card_id},
},
],
},
}
if self.source:
result["Source"] = {"type": "select", "select": {"name": self.source}}
return result
class Database:
data: Dict[str, Dict[str, Dict]]
def __init__(self):
self.data = dict()
def _retrieve_property(self, _type, _id, _property="name"):
try:
return self.data[_type][_id][_property]
except KeyError as e:
logging.error(f"Error trying to retrieve <{_type}.{_id}.{_property}>: {e} ")
return None
def _register(self, _type: str, _id: str, **kwargs):
if _type not in self.data:
self.data[_type] = dict()
self.data[_type][_id] = kwargs
def register_card(self, _id: str, **kwargs):
self._register("card", _id, **kwargs)
def register_card_type(self, _id: str, **kwargs):
self._register("card_type", _id, **kwargs)
def register_type(self, _id: str, **kwargs):
self._register("type", _id, **kwargs)
def register_rarity(self, _id: str, **kwargs):
self._register("rarity", _id, **kwargs)
def register_source(self, _id: str, **kwargs):
self._register("source", _id, **kwargs)
def register_color(self, _id: str, **kwargs):
self._register("color", _id, **kwargs)
def register_card(self, _id: str, **kwargs):
self._register("card", _id, **kwargs)
def iter_cards(self) -> Iterator:
for cid, card in self.data["card"].items():
yield Card(
global_id=card["gid"],
card_id=cid,
name=card["n"],
type=self._retrieve_property("type", card["t"]),
color=self._retrieve_property("color", card["col"]),
source=self._retrieve_property("source", card["cs"]),
rarity=self._retrieve_property("rarity", card["r"]),
image_url=card["iu"],
alternate_art=card["al"],
)

20
onepiece_tcg/notion.py Normal file
View File

@ -0,0 +1,20 @@
from notion_client import Client
class NotionClient:
client: Client
def __init__(self, token: str):
self.client = Client(auth=token)
def get_database_state(self, database_id: str):
return self.client.databases.query(database_id=database_id)
def update_card(self, page_id: str, **kwargs):
pass
# return self.client.databases.update(page_id, **kwargs)
def create_card(self, database_id: str, **kwargs):
return self.client.pages.create(
parent={"database_id": database_id}, properties=kwargs
)

154
poetry.lock generated Normal file
View File

@ -0,0 +1,154 @@
[[package]]
name = "anyio"
version = "3.6.1"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
category = "main"
optional = false
python-versions = ">=3.6.2"
[package.dependencies]
idna = ">=2.8"
sniffio = ">=1.1"
[package.extras]
trio = ["trio (>=0.16)"]
test = ["uvloop (>=0.15)", "mock (>=4)", "uvloop (<0.15)", "contextlib2", "trustme", "pytest-mock (>=3.6.1)", "pytest (>=7.0)", "hypothesis (>=4.0)", "coverage[toml] (>=4.5)"]
doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme", "packaging"]
[[package]]
name = "certifi"
version = "2022.6.15"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "h11"
version = "0.12.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "httpcore"
version = "0.15.0"
description = "A minimal low-level HTTP client."
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
anyio = ">=3.0.0,<4.0.0"
certifi = "*"
h11 = ">=0.11,<0.13"
sniffio = ">=1.0.0,<2.0.0"
[package.extras]
socks = ["socksio (>=1.0.0,<2.0.0)"]
http2 = ["h2 (>=3,<5)"]
[[package]]
name = "httpx"
version = "0.23.0"
description = "The next generation HTTP client."
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
certifi = "*"
httpcore = ">=0.15.0,<0.16.0"
rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
sniffio = "*"
[package.extras]
socks = ["socksio (>=1.0.0,<2.0.0)"]
http2 = ["h2 (>=3,<5)"]
cli = ["pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)", "click (>=8.0.0,<9.0.0)"]
brotli = ["brotli", "brotlicffi"]
[[package]]
name = "idna"
version = "3.3"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "notion-client"
version = "1.0.0"
description = "Python client for the official Notion API"
category = "main"
optional = false
python-versions = ">=3.7, <4"
[package.dependencies]
httpx = ">=0.15.0"
[[package]]
name = "rfc3986"
version = "1.5.0"
description = "Validating URI References per RFC 3986"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
[package.extras]
idna2008 = ["idna"]
[[package]]
name = "sniffio"
version = "1.2.0"
description = "Sniff out which async library your code is running under"
category = "main"
optional = false
python-versions = ">=3.5"
[metadata]
lock-version = "1.1"
python-versions = "^3.10"
content-hash = "53b08cfaa30a0b0ae66def96123c4fe2df15f759bf72130c56ff94fb7d0fbb00"
[metadata.files]
anyio = [
{file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"},
{file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"},
]
certifi = [
{file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
{file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
]
h11 = [
{file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
{file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
]
httpcore = [
{file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"},
{file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"},
]
httpx = [
{file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"},
{file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"},
]
idna = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
notion-client = [
{file = "notion-client-1.0.0.tar.gz", hash = "sha256:281b37a1512abeccb81377d4a3461833f1c1fbd0ceb6e681e38b7cd446bf1bce"},
{file = "notion_client-1.0.0-py2.py3-none-any.whl", hash = "sha256:01b3dfba071b5d09b6bdef2950624b7f000da2ece851c133ccf40815c4db167b"},
]
rfc3986 = [
{file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
{file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
]
sniffio = [
{file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"},
{file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"},
]

15
pyproject.toml Normal file
View File

@ -0,0 +1,15 @@
[tool.poetry]
name = "onepiece-tcg-notion-importer"
version = "0.1.0"
description = "Python script to update a notion database with the information from onepiece-cardgame.dev"
authors = ["Felipe Martin <me@fmartingr.com>"]
[tool.poetry.dependencies]
python = "^3.10"
notion-client = "^1.0.0"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@ -1,77 +0,0 @@
package main
import (
"encoding/json"
"time"
)
type isoDate struct {
time.Time
}
func (d *isoDate) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
t, _ := time.Parse("2006-01-02", s)
d.Time = t
return nil
}
func (d isoDate) MarshalJSON() ([]byte, error) {
return json.Marshal(d.Format("2006-01-02"))
}
// {"src_id":"0","name":"TBD","intl":"0","release_date":null,"imageURL":"","t":"0","filter":""}
type source struct {
SourceID string `json:"src_id"`
Name string `json:"name"`
International string `json:"intl"`
ReleaseDate isoDate `json:"release_date"`
ImageURL string `json:"image_url"`
T string `json:"t"`
Filter string `json:"filter"`
}
/* {
"gid": "47",
"cid": "OP01-047",
"n": "Trafalgar Law",
"t": "2",
"col": "7",
"cs": "5",
"tr": "Supernovas\/ Heart Pirates",
"a": "1",
"p": "6000",
"cp": null,
"l": null,
"r": "5",
"ar": null,
"iu": "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg",
"e": "<Blocker>\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.",
"al": null,
"intl": "0",
"srcN": "Romance Dawn [OP-01]",
"srcD": null
} */
type card struct {
GlobalID string `json:"gid"`
CardID string `json:"cid"`
Name string `json:"n"`
Type string `json:"t"`
Color string `json:"col"`
Source string `json:"cs"`
Tr string `json:"tr"`
AttackType string `json:"a"`
P string `json:"p"`
Cp string `json:"cp"`
L interface{} `json:"l"`
Rarity string `json:"r"`
Ar string `json:"ar"`
ImageURL string `json:"iu"`
E string `json:"e"`
Al interface{} `json:"al"`
International string `json:"intl"`
SourceName string `json:"srcN"`
SrcD interface{} `json:"srcD"`
}