diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..91b0a4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +*.pyc +*.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..ea46822 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# One Piece TCG Notion Importer + +Simple python script to import One Piece's TCG data from onepiece-card.dev to a notion database for collection tracking. diff --git a/const.go b/const.go deleted file mode 100644 index abaf225..0000000 --- a/const.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -const ( - sourcesURL = "https://onepiece-cardgame.dev/sources.json" - metaURL = "https://onepiece-cardgame.dev/meta.json" - cardsURL = "https://onepiece-cardgame.dev/cards.json" -) diff --git a/download.go b/download.go deleted file mode 100644 index d927dc5..0000000 --- a/download.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "io" - "log" - "net/http" -) - -func downloadURL(u string) ([]byte, error) { - log.Printf("Downloading %s\n", u) - - response, err := http.Get(u) - if err != nil { - return nil, err - } - - defer response.Body.Close() - - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return body, nil -} diff --git a/go.mod b/go.mod deleted file mode 100644 index 833dfcc..0000000 --- a/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module code.fmartingr.dev/fmartingr/onepiece-tcg-notion-importer - -go 1.18 - -require github.com/dstotijn/go-notion v0.6.1 diff --git a/go.sum b/go.sum deleted file mode 100644 index 4f9c641..0000000 --- a/go.sum +++ /dev/null @@ -1,5 +0,0 @@ -github.com/dstotijn/go-notion v0.6.1 h1:gmwU/JCdLC5szMasfysDOm8UG6/3P0bTUe0+CeW2fmI= -github.com/dstotijn/go-notion v0.6.1/go.mod h1:oxd+T9Wxduj5ZN7MRiHWtyGhGZLUFsUpZHMLS4uI1Qc= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go deleted file mode 100644 index 67ad878..0000000 --- a/main.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log" - - "github.com/dstotijn/go-notion" -) - -const ( - collectionsDatabaseID = "32fc86afa91e4718b17cb79ab8794265" -) - -func main() { - content, err := downloadURL(sourcesURL) - if err != nil { - panic(err) - } - - log.Println(string(content)) - - var sources []source - if err := json.Unmarshal(content, &sources); err != nil { - panic(err) - } - - ctx := context.Background() - dbSources := make(map[string]notion.Page) - - client := notion.NewClient("secret_135hSjxx1xcFWcrjcS1eejjp120T86V8sIGzVeW21X4") - - result, err := client.QueryDatabase(ctx, collectionsDatabaseID, ¬ion.DatabaseQuery{}) - if err != nil { - panic(err) - } - - for _, s := range result.Results { - dbSources[s.Properties.(notion.DatabasePageProperties)["SourceID"].Value().(string)] = s - } - - // for _, s := range sources { - - // } -} diff --git a/onepiece_tcg/__main__.py b/onepiece_tcg/__main__.py new file mode 100644 index 0000000..46f3bee --- /dev/null +++ b/onepiece_tcg/__main__.py @@ -0,0 +1,80 @@ +import logging +import os +import sys +from .notion import NotionClient +from .download import get_dict_from_url_response_body +from .models import Database + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + + try: + token = os.environ["NOTION_TOKEN"] + database_id = os.environ["NOTION_DATABASE_ID"] + card_id_property = "Card ID" + except KeyError as e: + logging.error("NOTION_TOKEN and NOTION_DATABASE_ID are required") + sys.exit(1) + + database = Database() + + # Download and build Meta database + logging.info("Downloading meta...") + meta = get_dict_from_url_response_body("https://onepiece-cardgame.dev/meta.json") + + # for attack in meta["a"]: + # database.register_attack(attack["atk_id"], **attack) + + for color in meta["c"]: + database.register_color(color["color_id"], **color) + + # for source in meta["s"]: + # database.register_source(source["src_id"], **source) + + for _type in meta["t"]: + database.register_type(_type["type_id"], **_type) + + for card_type in meta["ct"]: + database.register_card_type(card_type["type_id"], **card_type) + + for rarity in meta["r"]: + database.register_rarity(rarity["rarity_id"], **rarity) + + # for key in meta['key']: + # database.register_key(key['id'], **key) + + # Download and build Sources database + logging.info("Downloading sources...") + sources = get_dict_from_url_response_body( + "https://onepiece-cardgame.dev/sources.json" + ) + + for source in sources: + database.register_source(source["src_id"], **source) + + # Donwload and build Cards database + logging.info("Downloading cards...") + cards = get_dict_from_url_response_body("https://onepiece-cardgame.dev/cards.json") + + for card in cards: + database.register_card(card["cid"], **card) + + # Fetch current data from Notion + notion_cli = NotionClient(token) + notion_state = dict() + + try: + logging.info("Retrieving notion state...") + notion_db = notion_cli.get_database_state(database_id) + for card in notion_db["results"]: + notion_state[card[card_id_property]] = card + except Exception as e: + logging.error(e) + sys.exit(2) + + logging.info("Merging databases...") + for card in database.iter_cards(): + if card.card_id in notion_state: + notion_cli.update_card(notion_state[card.card_id], **{}) + else: + notion_cli.create_card(database_id, **card.to_notion()) diff --git a/onepiece_tcg/download.py b/onepiece_tcg/download.py new file mode 100644 index 0000000..3042b37 --- /dev/null +++ b/onepiece_tcg/download.py @@ -0,0 +1,20 @@ +import json +from urllib.request import urlopen, Request +from urllib.error import URLError + + +def get_dict_from_url_response_body(url: str) -> str: + try: + request = Request( + url, + headers={ + "User-Agent": "onepiece-tcg-notion-importer", + }, + ) + with urlopen(request) as response: + data = response.read() + return json.loads(data) + except URLError as e: + raise e + except json.JSONDecodeError as e: + raise e diff --git a/onepiece_tcg/models.py b/onepiece_tcg/models.py new file mode 100644 index 0000000..af70d3d --- /dev/null +++ b/onepiece_tcg/models.py @@ -0,0 +1,133 @@ +import json +import logging +from typing import Dict, Iterator + + +class Card(object): + global_id: str + card_id: str + name: str + type: str + color: str + source: str + rarity: str + image_url: str + alternate_art: str + + # gid: str # GlobalID: "47", + # cid: str # CardID: "OP01-047", + # n: str # Name: "Trafalgar Law", + # t: str # Type: "2", + # col: str # Color: "7", + # cs: str # Source: "5", + # tr: str # "Supernovas\/ Heart Pirates", + # a: str # AttackType: "1", + # p: str # Power: "6000", + # cp: str # null, + # l: dict # null, + # r: str # Rarity: "5", + # ar: str # null, + # iu: str # ImageURL: "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg", + # e: str # Effect: "\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.", + # al: dict # Alternate Art: "P1", + # intl: str # International: "0", + # srcN: str # SourceName: "Romance Dawn [OP-01]", + # srcD: dict # null + + def __init__( + self, + global_id, + card_id, + name, + type, + color, + source, + rarity, + image_url, + alternate_art, + ): + self.global_id = global_id + self.card_id = card_id + self.name = name + self.type = type + self.color = color + self.source = source + self.rarity = rarity + self.image_url = image_url + self.alternate_art = alternate_art + + def __repr__(self): + return json.dumps(self.__dict__) + + def to_notion(self): + result = { + "Name": {"title": [{"text": {"content": self.name}}]}, + "Rarity": {"type": "select", "select": {"name": self.rarity}}, + "Card ID": { + "type": "rich_text", + "rich_text": [ + { + "type": "text", + "text": {"content": self.card_id}, + }, + ], + }, + } + + if self.source: + result["Source"] = {"type": "select", "select": {"name": self.source}} + return result + + +class Database: + data: Dict[str, Dict[str, Dict]] + + def __init__(self): + self.data = dict() + + def _retrieve_property(self, _type, _id, _property="name"): + try: + return self.data[_type][_id][_property] + except KeyError as e: + logging.error(f"Error trying to retrieve <{_type}.{_id}.{_property}>: {e} ") + return None + + def _register(self, _type: str, _id: str, **kwargs): + if _type not in self.data: + self.data[_type] = dict() + self.data[_type][_id] = kwargs + + def register_card(self, _id: str, **kwargs): + self._register("card", _id, **kwargs) + + def register_card_type(self, _id: str, **kwargs): + self._register("card_type", _id, **kwargs) + + def register_type(self, _id: str, **kwargs): + self._register("type", _id, **kwargs) + + def register_rarity(self, _id: str, **kwargs): + self._register("rarity", _id, **kwargs) + + def register_source(self, _id: str, **kwargs): + self._register("source", _id, **kwargs) + + def register_color(self, _id: str, **kwargs): + self._register("color", _id, **kwargs) + + def register_card(self, _id: str, **kwargs): + self._register("card", _id, **kwargs) + + def iter_cards(self) -> Iterator: + for cid, card in self.data["card"].items(): + yield Card( + global_id=card["gid"], + card_id=cid, + name=card["n"], + type=self._retrieve_property("type", card["t"]), + color=self._retrieve_property("color", card["col"]), + source=self._retrieve_property("source", card["cs"]), + rarity=self._retrieve_property("rarity", card["r"]), + image_url=card["iu"], + alternate_art=card["al"], + ) diff --git a/onepiece_tcg/notion.py b/onepiece_tcg/notion.py new file mode 100644 index 0000000..e925560 --- /dev/null +++ b/onepiece_tcg/notion.py @@ -0,0 +1,20 @@ +from notion_client import Client + + +class NotionClient: + client: Client + + def __init__(self, token: str): + self.client = Client(auth=token) + + def get_database_state(self, database_id: str): + return self.client.databases.query(database_id=database_id) + + def update_card(self, page_id: str, **kwargs): + pass + # return self.client.databases.update(page_id, **kwargs) + + def create_card(self, database_id: str, **kwargs): + return self.client.pages.create( + parent={"database_id": database_id}, properties=kwargs + ) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..29f26fe --- /dev/null +++ b/poetry.lock @@ -0,0 +1,154 @@ +[[package]] +name = "anyio" +version = "3.6.1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +trio = ["trio (>=0.16)"] +test = ["uvloop (>=0.15)", "mock (>=4)", "uvloop (<0.15)", "contextlib2", "trustme", "pytest-mock (>=3.6.1)", "pytest (>=7.0)", "hypothesis (>=4.0)", "coverage[toml] (>=4.5)"] +doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme", "packaging"] + +[[package]] +name = "certifi" +version = "2022.6.15" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "h11" +version = "0.12.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "httpcore" +version = "0.15.0" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +anyio = ">=3.0.0,<4.0.0" +certifi = "*" +h11 = ">=0.11,<0.13" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +socks = ["socksio (>=1.0.0,<2.0.0)"] +http2 = ["h2 (>=3,<5)"] + +[[package]] +name = "httpx" +version = "0.23.0" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.16.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +socks = ["socksio (>=1.0.0,<2.0.0)"] +http2 = ["h2 (>=3,<5)"] +cli = ["pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)", "click (>=8.0.0,<9.0.0)"] +brotli = ["brotli", "brotlicffi"] + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "notion-client" +version = "1.0.0" +description = "Python client for the official Notion API" +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +httpx = ">=0.15.0" + +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "sniffio" +version = "1.2.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.5" + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "53b08cfaa30a0b0ae66def96123c4fe2df15f759bf72130c56ff94fb7d0fbb00" + +[metadata.files] +anyio = [ + {file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"}, + {file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"}, +] +certifi = [ + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, +] +h11 = [ + {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, + {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, +] +httpcore = [ + {file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"}, + {file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"}, +] +httpx = [ + {file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"}, + {file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"}, +] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] +notion-client = [ + {file = "notion-client-1.0.0.tar.gz", hash = "sha256:281b37a1512abeccb81377d4a3461833f1c1fbd0ceb6e681e38b7cd446bf1bce"}, + {file = "notion_client-1.0.0-py2.py3-none-any.whl", hash = "sha256:01b3dfba071b5d09b6bdef2950624b7f000da2ece851c133ccf40815c4db167b"}, +] +rfc3986 = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] +sniffio = [ + {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"}, + {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c82d0f5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "onepiece-tcg-notion-importer" +version = "0.1.0" +description = "Python script to update a notion database with the information from onepiece-cardgame.dev" +authors = ["Felipe Martin "] + +[tool.poetry.dependencies] +python = "^3.10" +notion-client = "^1.0.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/types.go b/types.go deleted file mode 100644 index 21c7a20..0000000 --- a/types.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "encoding/json" - "time" -) - -type isoDate struct { - time.Time -} - -func (d *isoDate) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - t, _ := time.Parse("2006-01-02", s) - d.Time = t - return nil -} -func (d isoDate) MarshalJSON() ([]byte, error) { - return json.Marshal(d.Format("2006-01-02")) -} - -// {"src_id":"0","name":"TBD","intl":"0","release_date":null,"imageURL":"","t":"0","filter":""} -type source struct { - SourceID string `json:"src_id"` - Name string `json:"name"` - International string `json:"intl"` - ReleaseDate isoDate `json:"release_date"` - ImageURL string `json:"image_url"` - T string `json:"t"` - Filter string `json:"filter"` -} - -/* { - "gid": "47", - "cid": "OP01-047", - "n": "Trafalgar Law", - "t": "2", - "col": "7", - "cs": "5", - "tr": "Supernovas\/ Heart Pirates", - "a": "1", - "p": "6000", - "cp": null, - "l": null, - "r": "5", - "ar": null, - "iu": "https:\/\/onepiece-cardgame.dev\/images\/cards\/OP01-047_616aca_jp.jpg", - "e": "\r\n[On Play] You may return one of your Characters to your hand: Play 1 Cost 3 or lower Character Card from your hand.", - "al": null, - "intl": "0", - "srcN": "Romance Dawn [OP-01]", - "srcD": null -} */ -type card struct { - GlobalID string `json:"gid"` - CardID string `json:"cid"` - Name string `json:"n"` - Type string `json:"t"` - Color string `json:"col"` - Source string `json:"cs"` - Tr string `json:"tr"` - AttackType string `json:"a"` - P string `json:"p"` - Cp string `json:"cp"` - L interface{} `json:"l"` - Rarity string `json:"r"` - Ar string `json:"ar"` - ImageURL string `json:"iu"` - E string `json:"e"` - Al interface{} `json:"al"` - International string `json:"intl"` - SourceName string `json:"srcN"` - SrcD interface{} `json:"srcD"` -}