diff --git a/minecraftcodex/database/migrations/0011_auto__add_language__add_languagestring.py b/minecraftcodex/database/migrations/0011_auto__add_language__add_languagestring.py new file mode 100644 index 0000000..b8ffe1c --- /dev/null +++ b/minecraftcodex/database/migrations/0011_auto__add_language__add_languagestring.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Language' + db.create_table(u'database_language', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('region', self.gf('django.db.models.fields.CharField')(max_length=32)), + ('code', self.gf('django.db.models.fields.CharField')(max_length=12)), + )) + db.send_create_signal(u'database', ['Language']) + + # Adding model 'LanguageString' + db.create_table(u'database_languagestring', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('language', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['database.Language'])), + ('key', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('value', self.gf('django.db.models.fields.CharField')(max_length=512)), + )) + db.send_create_signal(u'database', ['LanguageString']) + + + def backwards(self, orm): + # Deleting model 'Language' + db.delete_table(u'database_language') + + # Deleting model 'LanguageString' + db.delete_table(u'database_languagestring') + + + models = { + u'database.block': { + 'Meta': {'object_name': 'Block'}, + 'data_value': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'}) + }, + u'database.item': { + 'Meta': {'object_name': 'Item'}, + 'data_value': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'}) + }, + 'database.jarfile': { + 'Meta': {'object_name': 'JarFile'}, + 'description': ('django.db.models.fields.CharField', [], {'default': "'client'", 'max_length': '256'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Version']"}) + }, + u'database.language': { + 'Meta': {'object_name': 'Language'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '12'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'region': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + u'database.languagestring': { + 'Meta': {'object_name': 'LanguageString'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'language': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Language']"}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'database.mod': { + 'Meta': {'ordering': "['name']", 'object_name': 'Mod'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}) + }, + u'database.texture': { + 'Meta': {'object_name': 'Texture'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'type': ('django.db.models.fields.CharField', [], {'default': "'items'", 'max_length': '16'}) + }, + 'database.version': { + 'Meta': {'ordering': "['date']", 'object_name': 'Version'}, + 'changelog': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Mod']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'snapshot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'release'", 'max_length': '10', 'blank': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'version_number': ('django.db.models.fields.CharField', [], {'default': "'0.1.0'", 'max_length': '256'}) + } + } + + complete_apps = ['database'] \ No newline at end of file diff --git a/minecraftcodex/database/migrations/0012_auto.py b/minecraftcodex/database/migrations/0012_auto.py new file mode 100644 index 0000000..010ec8d --- /dev/null +++ b/minecraftcodex/database/migrations/0012_auto.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding index on 'Language', fields ['name'] + db.create_index(u'database_language', ['name']) + + # Adding index on 'LanguageString', fields ['key'] + db.create_index(u'database_languagestring', ['key']) + + + def backwards(self, orm): + # Removing index on 'LanguageString', fields ['key'] + db.delete_index(u'database_languagestring', ['key']) + + # Removing index on 'Language', fields ['name'] + db.delete_index(u'database_language', ['name']) + + + models = { + u'database.block': { + 'Meta': {'object_name': 'Block'}, + 'data_value': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'}) + }, + u'database.item': { + 'Meta': {'object_name': 'Item'}, + 'data_value': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'}) + }, + 'database.jarfile': { + 'Meta': {'object_name': 'JarFile'}, + 'description': ('django.db.models.fields.CharField', [], {'default': "'client'", 'max_length': '256'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Version']"}) + }, + u'database.language': { + 'Meta': {'object_name': 'Language'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '12'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'db_index': 'True'}), + 'region': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + u'database.languagestring': { + 'Meta': {'object_name': 'LanguageString'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '256', 'db_index': 'True'}), + 'language': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Language']"}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'database.mod': { + 'Meta': {'ordering': "['name']", 'object_name': 'Mod'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}) + }, + u'database.texture': { + 'Meta': {'object_name': 'Texture'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'type': ('django.db.models.fields.CharField', [], {'default': "'items'", 'max_length': '16'}) + }, + 'database.version': { + 'Meta': {'ordering': "['date']", 'object_name': 'Version'}, + 'changelog': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Mod']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'snapshot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'release'", 'max_length': '10', 'blank': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'version_number': ('django.db.models.fields.CharField', [], {'default': "'0.1.0'", 'max_length': '256'}) + } + } + + complete_apps = ['database'] \ No newline at end of file diff --git a/minecraftcodex/database/models.py b/minecraftcodex/database/models.py index 46259a9..4663707 100644 --- a/minecraftcodex/database/models.py +++ b/minecraftcodex/database/models.py @@ -158,6 +158,18 @@ class Item(models.Model): main_texture = models.ForeignKey('Texture', null=True) data_value = models.IntegerField() + def name(self): + result = self.internal_name + try: + string = LanguageString.objects.get( + language=14, + key='item.%s.name' % self.internal_name + ) + result = string.value + except: + pass + return result + class ItemAdmin(admin.ModelAdmin): list_display = ('internal_name', 'data_value', 'main_texture_html') @@ -185,6 +197,18 @@ class Block(models.Model): main_texture = models.ForeignKey('Texture', null=True) data_value = models.IntegerField() + def name(self): + result = self.internal_name + try: + string = LanguageString.objects.get( + language=14, + key='tile.%s.name' % self.internal_name + ) + result = string.value + except: + pass + return result + class BlockAdmin(admin.ModelAdmin): list_display = ('internal_name', 'data_value', 'main_texture_html') @@ -202,3 +226,39 @@ class BlockAdmin(admin.ModelAdmin): main_texture_html.allow_tags = True admin.site.register(Block, BlockAdmin) + + +### +# LANGUAGES +### +class Language(models.Model): + name = models.CharField(max_length=64, db_index=True) + region = models.CharField(max_length=32) + code = models.CharField(max_length=12) + + def __unicode__(self): + return "%s (%s)" % (self.name, self.region) + + +class LanguageAdmin(admin.ModelAdmin): + list_display = ('name', 'region', ) + list_display_links = ('name', ) + search_fields = ('name', ) + +admin.site.register(Language, LanguageAdmin) + + +class LanguageString(models.Model): + language = models.ForeignKey('Language', db_index=True) + key = models.CharField(max_length=256, db_index=True) + value = models.CharField(max_length=512) + + +class LanguageStringAdmin(admin.ModelAdmin): + list_display = ('language', 'key', 'value', ) + list_display_links = ('language', 'key', ) + + list_filter = ('language', ) + search_fields = ('key', 'value', ) + +admin.site.register(LanguageString, LanguageStringAdmin) diff --git a/minecraftcodex/database/templates/blocks.html b/minecraftcodex/database/templates/blocks.html index f5e38c7..3835041 100644 --- a/minecraftcodex/database/templates/blocks.html +++ b/minecraftcodex/database/templates/blocks.html @@ -25,7 +25,7 @@ - {{ item.internal_name }} + {{ item.name() }} {% endfor %} diff --git a/minecraftcodex/database/templates/items.html b/minecraftcodex/database/templates/items.html index 99e6a60..9f929aa 100644 --- a/minecraftcodex/database/templates/items.html +++ b/minecraftcodex/database/templates/items.html @@ -25,7 +25,7 @@ - {{ item.internal_name }} + {{ item.name() }} {% endfor %} diff --git a/utils/miner/conf.py b/utils/miner/conf.py index 43d1efd..f96c802 100644 --- a/utils/miner/conf.py +++ b/utils/miner/conf.py @@ -1,5 +1,6 @@ DEBUG = False +# Save to database? SAVE = True ### @@ -24,6 +25,17 @@ BLOCKS_FILES = [] BLOCKS_JAVA_KEYWORDS = ['stonebrick'] BLOCKS_PATTERN = "new (?P[a-z]{1,3}\((?P[1-9]{1,3}).*\"(?P\w+)\"\))" +### +# LANGUAGES +### +LANGUAGES_PATH = 'jarfile/lang' +LANGUAGES_MASTER_KEYS = [ + 'language.name', + 'language.region', + 'language.code', +] + + ### # BLACKLIST ### diff --git a/utils/miner/languages.py b/utils/miner/languages.py new file mode 100644 index 0000000..a0d2ca7 --- /dev/null +++ b/utils/miner/languages.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python + +# General libs +import re +import json +import os +import sys + +# Tool libs +from utils import run, sanitize +import conf +from objects import GameLanguage + + +print("=> Phase: languages") +if conf.SAVE: + sys.path.append('../../minecraftcodex') + os.environ['DJANGO_SETTINGS_MODULE'] = 'local_settings' + from database.models import Language, LanguageString + +### +# GLOBALS +### +STRINGS = [] +LANGUAGES = [] +LANGUAGES_STR = [] + +### +# LOOK FOR CORRECT JAVA FILES +### +print(" => Looking for languages files...") +directory_list = os.listdir(conf.LANGUAGES_PATH) +print(" Found %d file(s)." % len(directory_list)) + +### +# GET LANGUAGES +### +try: + OLD_STRINGS = json.loads(open('strings.json').read()) +except: + OLD_STRINGS = [] +try: + OLD_LANGUAGES = json.loads(open('languages.json').read()) +except: + OLD_LANGUAGES = [] + +print(" => Mining languages...") +for item in directory_list: + if '.lang' in item: + if conf.DEBUG: + print(" Now %s " % item) + + language = open('%s/%s' % (conf.LANGUAGES_PATH, item)) + language_obj = GameLanguage() + for line in language.readlines(): + line = line.strip() + if line and 'X-Generator' not in line: + key, value = line.split('=', 1) + if key in conf.LANGUAGES_MASTER_KEYS: + # Language object + setattr(language_obj, key.split('.')[1], value) + else: + # Language String object + language_obj.add_string(key, value) + + # Store for comparision + if language_obj.code == 'en_US': + if key not in STRINGS: + STRINGS.append(key) + LANGUAGES.append(language_obj) + +if conf.SAVE: + for item in LANGUAGES: + try: + obj = Language.objects.get( + name=item.name, + region=item.region, + code=item.code + ) + except Language.DoesNotExist: + obj = Language( + name=item.name, + region=item.region, + code=item.code + ) + obj.save() + for key in item.strings.keys(): + value = item.strings[key] + try: + string_obj = LanguageString.objects.get( + language=obj, + key=key + ) + if string_obj.value != value: + string_obj.value = value + string_obj.save() + except LanguageString.DoesNotExist: + string_obj = LanguageString( + language=obj, + key=key, + value=value + ) + string_obj.save() + + + +print(" => Summary") + +# LANGUAGES +[LANGUAGES_STR.append(x.name) for x in LANGUAGES] +new_languages = len(LANGUAGES_STR) - len(OLD_LANGUAGES) +print(" Found %d languages (%d new)." % (len(LANGUAGES_STR), new_languages)) +if len(LANGUAGES_STR) != len(OLD_LANGUAGES): + print(" Comparision:") + + for string in LANGUAGES_STR: + if string not in OLD_LANGUAGES: + print(" + %s" % string) + + for string in OLD_LANGUAGES: + if string not in LANGUAGES_STR: + print(" - %s" % string) + +olditems = open('languages.json', 'w') +olditems.write(json.dumps(LANGUAGES_STR)) +olditems.close() + +# STRINGS +new_strings = len(STRINGS) - len(OLD_STRINGS) +print(" Found %d strings (%d new) -based on en_US-." % (len(STRINGS), new_strings)) +if len(STRINGS) != len(OLD_STRINGS): + print(" Comparision:") + + for string in STRINGS: + if string not in OLD_STRINGS: + print(" + %s" % string) + + for string in OLD_STRINGS: + if string not in STRINGS: + print(" - %s" % string) + +olditems = open('strings.json', 'w') +olditems.write(json.dumps(STRINGS)) +olditems.close() diff --git a/utils/miner/objects.py b/utils/miner/objects.py index 3fce9f1..6c2e576 100644 --- a/utils/miner/objects.py +++ b/utils/miner/objects.py @@ -59,3 +59,24 @@ class GameTexture(object): def parse_name(self, name): return name.split('.')[0] + + +### +# LANGUAGES +### +class GameLanguage(object): + def __init__(self, name=None, region=None, code=None): + self.name = name + self.region = region + self.code = code + self.strings = {} + + def __str__(self): + return "" % ( + self.code, + self.name + ) + + def add_string(self, key, value): + if key not in self.strings: + self.strings[key] = value diff --git a/utils/miner/run.sh b/utils/miner/run.sh index 458e670..21a7425 100755 --- a/utils/miner/run.sh +++ b/utils/miner/run.sh @@ -7,3 +7,5 @@ python textures.py python items.py python blocks.py + +python languages.py