From 29401d6bea791e9d9a79f37a87f62159014444df Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sat, 20 Aug 2022 14:43:28 +0200 Subject: [PATCH] moved ty python --- .gitignore | 3 + README.md | 3 + const.go | 7 -- download.go | 25 ------- go.mod | 5 -- go.sum | 5 -- main.go | 45 ------------ onepiece_tcg/__main__.py | 80 ++++++++++++++++++++ onepiece_tcg/download.py | 20 +++++ onepiece_tcg/models.py | 133 +++++++++++++++++++++++++++++++++ onepiece_tcg/notion.py | 20 +++++ poetry.lock | 154 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 15 ++++ types.go | 77 -------------------- 14 files changed, 428 insertions(+), 164 deletions(-) create mode 100644 .gitignore create mode 100644 README.md delete mode 100644 const.go delete mode 100644 download.go delete mode 100644 go.mod delete mode 100644 go.sum delete mode 100644 main.go create mode 100644 onepiece_tcg/__main__.py create mode 100644 onepiece_tcg/download.py create mode 100644 onepiece_tcg/models.py create mode 100644 onepiece_tcg/notion.py create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100644 types.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..91b0a4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +*.pyc +*.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..ea46822 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# One Piece TCG Notion Importer + +Simple python script to import One Piece's TCG data from onepiece-card.dev to a notion database for collection tracking. diff --git a/const.go b/const.go deleted file mode 100644 index abaf225..0000000 --- a/const.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -const ( - sourcesURL = "https://onepiece-cardgame.dev/sources.json" - metaURL = "https://onepiece-cardgame.dev/meta.json" - cardsURL = "https://onepiece-cardgame.dev/cards.json" -) diff --git a/download.go b/download.go deleted file mode 100644 index d927dc5..0000000 --- a/download.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "io" - "log" - "net/http" -) - -func downloadURL(u string) ([]byte, error) { - log.Printf("Downloading %s\n", u) - - response, err := http.Get(u) - if err != nil { - return nil, err - } - - defer response.Body.Close() - - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return body, nil -} diff --git a/go.mod b/go.mod deleted file mode 100644 index 833dfcc..0000000 --- a/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module code.fmartingr.dev/fmartingr/onepiece-tcg-notion-importer - -go 1.18 - -require github.com/dstotijn/go-notion v0.6.1 diff --git a/go.sum b/go.sum deleted file mode 100644 index 4f9c641..0000000 --- a/go.sum +++ /dev/null @@ -1,5 +0,0 @@ -github.com/dstotijn/go-notion v0.6.1 h1:gmwU/JCdLC5szMasfysDOm8UG6/3P0bTUe0+CeW2fmI= -github.com/dstotijn/go-notion v0.6.1/go.mod h1:oxd+T9Wxduj5ZN7MRiHWtyGhGZLUFsUpZHMLS4uI1Qc= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go deleted file mode 100644 index 67ad878..0000000 --- a/main.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log" - - "github.com/dstotijn/go-notion" -) - -const ( - collectionsDatabaseID = "32fc86afa91e4718b17cb79ab8794265" -) - -func main() { - content, err := downloadURL(sourcesURL) - if err != nil { - panic(err) - } - - log.Println(string(content)) - - var sources []source - if err := json.Unmarshal(content, &sources); err != nil { - panic(err) - } - - ctx := context.Background() - dbSources := make(map[string]notion.Page) - - client := notion.NewClient("secret_135hSjxx1xcFWcrjcS1eejjp120T86V8sIGzVeW21X4") - - result, err := client.QueryDatabase(ctx, collectionsDatabaseID, ¬ion.DatabaseQuery{}) - if err != nil { - panic(err) - } - - for _, s := range result.Results { - dbSources[s.Properties.(notion.DatabasePageProperties)["SourceID"].Value().(string)] = s - } - - // for _, s := range sources { - - // } -} diff --git a/onepiece_tcg/__main__.py b/onepiece_tcg/__main__.py new file mode 100644 index 0000000..46f3bee --- /dev/null +++ b/onepiece_tcg/__main__.py @@ -0,0 +1,80 @@ +import logging +import os +import sys +from .notion import NotionClient +from .download import get_dict_from_url_response_body +from .models import Database + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + + try: + token = os.environ["NOTION_TOKEN"] + database_id = os.environ["NOTION_DATABASE_ID"] + card_id_property = "Card ID" + except KeyError as e: + logging.error("NOTION_TOKEN and NOTION_DATABASE_ID are required") + sys.exit(1) + + database = Database() + + # Download and build Meta database + logging.info("Downloading meta...") + meta = get_dict_from_url_response_body("https://onepiece-cardgame.dev/meta.json") + + # for attack in meta["a"]: + # database.register_attack(attack["atk_id"], **attack) + + for color in meta["c"]: + database.register_color(color["color_id"], **color) + + # for source in meta["s"]: + # database.register_source(source["src_id"], **source) + + for _type in meta["t"]: + database.register_type(_type["type_id"], **_type) + + for card_type in meta["ct"]: + database.register_card_type(card_type["type_id"], **card_type) + + for rarity in meta["r"]: + database.register_rarity(rarity["rarity_id"], **rarity) + + # for key in meta['key']: + # database.register_key(key['id'], **key) + + # Download and build Sources database + logging.info("Downloading sources...") + sources = get_dict_from_url_response_body( + "https://onepiece-cardgame.dev/sources.json" + ) + + for source in sources: + database.register_source(source["src_id"], **source) + + # Donwload and build Cards database + logging.info("Downloading cards...") + cards = get_dict_from_url_response_body("https://onepiece-cardgame.dev/cards.json") + + for card in cards: + database.register_card(card["cid"], **card) + + # Fetch current data from Notion + notion_cli = NotionClient(token) + notion_state = dict() + + try: + logging.info("Retrieving notion state...") + notion_db = notion_cli.get_database_state(database_id) + for card in notion_db["results"]: + notion_state[card[card_id_property]] = card + except Exception as e: + logging.error(e) + sys.exit(2) + + logging.info("Merging databases...") + for card in database.iter_cards(): + if card.card_id in notion_state: + notion_cli.update_card(notion_state[card.card_id], **{}) + else: + notion_cli.create_card(database_id, **card.to_notion()) diff --git a/onepiece_tcg/download.py b/onepiece_tcg/download.py new file mode 100644 index 0000000..3042b37 --- /dev/null +++ b/onepiece_tcg/download.py @@ -0,0 +1,20 @@ +import json +from urllib.request import urlopen, Request +from urllib.error import URLError + + +def get_dict_from_url_response_body(url: str) -> str: + try: + request = Request( + url, + headers={ + "User-Agent": "onepiece-tcg-notion-importer", + }, + ) + with urlopen(request) as response: + data = response.read() + return json.loads(data) + except URLError as e: + raise e + except json.JSONDecodeError as e: + raise e diff --git a/onepiece_tcg/models.py b/onepiece_tcg/models.py new file mode 100644 index 0000000..af70d3d --- /dev/null +++ b/onepiece_tcg/models.py @@ -0,0 +1,133 @@ +import json +import logging +from typing import Dict, Iterator + + +class Card(object): + global_id: str + card_id: str + name: str + type: str + color: str + source: str + rarity: str + image_url: str + alternate_art: str + + # gid: str # GlobalID: "47", + # cid: str # CardID: "OP01-047", + # n: str # Name: "Trafalgar Law", + # t: str # Type: "2", + # col: str # Color: "7", + # cs: str # Source: "5", + # tr: str # "Supernovas\/ Heart Pirates", + # a: str # AttackType: "1", + # p: str # Power: "6000", + # cp: str # null, + # l: dict # null, + # r: str # Rarity: "5", + # ar: str # null, + # iu: str # ImageURL: "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg", + # e: str # Effect: "\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.", + # al: dict # Alternate Art: "P1", + # intl: str # International: "0", + # srcN: str # SourceName: "Romance Dawn [OP-01]", + # srcD: dict # null + + def __init__( + self, + global_id, + card_id, + name, + type, + color, + source, + rarity, + image_url, + alternate_art, + ): + self.global_id = global_id + self.card_id = card_id + self.name = name + self.type = type + self.color = color + self.source = source + self.rarity = rarity + self.image_url = image_url + self.alternate_art = alternate_art + + def __repr__(self): + return json.dumps(self.__dict__) + + def to_notion(self): + result = { + "Name": {"title": [{"text": {"content": self.name}}]}, + "Rarity": {"type": "select", "select": {"name": self.rarity}}, + "Card ID": { + "type": "rich_text", + "rich_text": [ + { + "type": "text", + "text": {"content": self.card_id}, + }, + ], + }, + } + + if self.source: + result["Source"] = {"type": "select", "select": {"name": self.source}} + return result + + +class Database: + data: Dict[str, Dict[str, Dict]] + + def __init__(self): + self.data = dict() + + def _retrieve_property(self, _type, _id, _property="name"): + try: + return self.data[_type][_id][_property] + except KeyError as e: + logging.error(f"Error trying to retrieve <{_type}.{_id}.{_property}>: {e} ") + return None + + def _register(self, _type: str, _id: str, **kwargs): + if _type not in self.data: + self.data[_type] = dict() + self.data[_type][_id] = kwargs + + def register_card(self, _id: str, **kwargs): + self._register("card", _id, **kwargs) + + def register_card_type(self, _id: str, **kwargs): + self._register("card_type", _id, **kwargs) + + def register_type(self, _id: str, **kwargs): + self._register("type", _id, **kwargs) + + def register_rarity(self, _id: str, **kwargs): + self._register("rarity", _id, **kwargs) + + def register_source(self, _id: str, **kwargs): + self._register("source", _id, **kwargs) + + def register_color(self, _id: str, **kwargs): + self._register("color", _id, **kwargs) + + def register_card(self, _id: str, **kwargs): + self._register("card", _id, **kwargs) + + def iter_cards(self) -> Iterator: + for cid, card in self.data["card"].items(): + yield Card( + global_id=card["gid"], + card_id=cid, + name=card["n"], + type=self._retrieve_property("type", card["t"]), + color=self._retrieve_property("color", card["col"]), + source=self._retrieve_property("source", card["cs"]), + rarity=self._retrieve_property("rarity", card["r"]), + image_url=card["iu"], + alternate_art=card["al"], + ) diff --git a/onepiece_tcg/notion.py b/onepiece_tcg/notion.py new file mode 100644 index 0000000..e925560 --- /dev/null +++ b/onepiece_tcg/notion.py @@ -0,0 +1,20 @@ +from notion_client import Client + + +class NotionClient: + client: Client + + def __init__(self, token: str): + self.client = Client(auth=token) + + def get_database_state(self, database_id: str): + return self.client.databases.query(database_id=database_id) + + def update_card(self, page_id: str, **kwargs): + pass + # return self.client.databases.update(page_id, **kwargs) + + def create_card(self, database_id: str, **kwargs): + return self.client.pages.create( + parent={"database_id": database_id}, properties=kwargs + ) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..29f26fe --- /dev/null +++ b/poetry.lock @@ -0,0 +1,154 @@ +[[package]] +name = "anyio" +version = "3.6.1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +trio = ["trio (>=0.16)"] +test = ["uvloop (>=0.15)", "mock (>=4)", "uvloop (<0.15)", "contextlib2", "trustme", "pytest-mock (>=3.6.1)", "pytest (>=7.0)", "hypothesis (>=4.0)", "coverage[toml] (>=4.5)"] +doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme", "packaging"] + +[[package]] +name = "certifi" +version = "2022.6.15" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "h11" +version = "0.12.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "httpcore" +version = "0.15.0" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +anyio = ">=3.0.0,<4.0.0" +certifi = "*" +h11 = ">=0.11,<0.13" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +socks = ["socksio (>=1.0.0,<2.0.0)"] +http2 = ["h2 (>=3,<5)"] + +[[package]] +name = "httpx" +version = "0.23.0" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.16.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +socks = ["socksio (>=1.0.0,<2.0.0)"] +http2 = ["h2 (>=3,<5)"] +cli = ["pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)", "click (>=8.0.0,<9.0.0)"] +brotli = ["brotli", "brotlicffi"] + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "notion-client" +version = "1.0.0" +description = "Python client for the official Notion API" +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +httpx = ">=0.15.0" + +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "sniffio" +version = "1.2.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.5" + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "53b08cfaa30a0b0ae66def96123c4fe2df15f759bf72130c56ff94fb7d0fbb00" + +[metadata.files] +anyio = [ + {file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"}, + {file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"}, +] +certifi = [ + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, +] +h11 = [ + {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, + {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, +] +httpcore = [ + {file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"}, + {file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"}, +] +httpx = [ + {file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"}, + {file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"}, +] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] +notion-client = [ + {file = "notion-client-1.0.0.tar.gz", hash = "sha256:281b37a1512abeccb81377d4a3461833f1c1fbd0ceb6e681e38b7cd446bf1bce"}, + {file = "notion_client-1.0.0-py2.py3-none-any.whl", hash = "sha256:01b3dfba071b5d09b6bdef2950624b7f000da2ece851c133ccf40815c4db167b"}, +] +rfc3986 = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] +sniffio = [ + {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"}, + {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c82d0f5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "onepiece-tcg-notion-importer" +version = "0.1.0" +description = "Python script to update a notion database with the information from onepiece-cardgame.dev" +authors = ["Felipe Martin "] + +[tool.poetry.dependencies] +python = "^3.10" +notion-client = "^1.0.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/types.go b/types.go deleted file mode 100644 index 21c7a20..0000000 --- a/types.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "encoding/json" - "time" -) - -type isoDate struct { - time.Time -} - -func (d *isoDate) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - t, _ := time.Parse("2006-01-02", s) - d.Time = t - return nil -} -func (d isoDate) MarshalJSON() ([]byte, error) { - return json.Marshal(d.Format("2006-01-02")) -} - -// {"src_id":"0","name":"TBD","intl":"0","release_date":null,"imageURL":"","t":"0","filter":""} -type source struct { - SourceID string `json:"src_id"` - Name string `json:"name"` - International string `json:"intl"` - ReleaseDate isoDate `json:"release_date"` - ImageURL string `json:"image_url"` - T string `json:"t"` - Filter string `json:"filter"` -} - -/* { - "gid": "47", - "cid": "OP01-047", - "n": "Trafalgar Law", - "t": "2", - "col": "7", - "cs": "5", - "tr": "Supernovas\/ Heart Pirates", - "a": "1", - "p": "6000", - "cp": null, - "l": null, - "r": "5", - "ar": null, - "iu": "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg", - "e": "\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.", - "al": null, - "intl": "0", - "srcN": "Romance Dawn [OP-01]", - "srcD": null -} */ -type card struct { - GlobalID string `json:"gid"` - CardID string `json:"cid"` - Name string `json:"n"` - Type string `json:"t"` - Color string `json:"col"` - Source string `json:"cs"` - Tr string `json:"tr"` - AttackType string `json:"a"` - P string `json:"p"` - Cp string `json:"cp"` - L interface{} `json:"l"` - Rarity string `json:"r"` - Ar string `json:"ar"` - ImageURL string `json:"iu"` - E string `json:"e"` - Al interface{} `json:"al"` - International string `json:"intl"` - SourceName string `json:"srcN"` - SrcD interface{} `json:"srcD"` -}