fmartingr
/
minecraftcodex
mirror of https://github.com/fmartingr/minecraftcodex.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'miner-languages' into develop 

* miner-languages:
  Blocks and Items now have a name() method who searches for the correct key to obtain its name. Default to ID 14 = English US
  Finished languages miner
  Added Language and LanguageString models
  Added language step to the miner! KEEP IN MIND THAT SNAPSHOT ONLY HAVE ENGLISH!
This commit is contained in:
Felipe Martín 2013-05-28 16:51:42 +02:00
parent a19b8501be 301686a7f3
commit f3c034efb9
9 changed files with 431 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
minecraftcodex/database/migrations/0011_auto__add_language__add_languagestring.py Normal file
View File

@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'Language'
db.create_table(u'database_language', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=64)),
('region', self.gf('django.db.models.fields.CharField')(max_length=32)),
('code', self.gf('django.db.models.fields.CharField')(max_length=12)),
))
db.send_create_signal(u'database', ['Language'])
# Adding model 'LanguageString'
db.create_table(u'database_languagestring', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('language', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['database.Language'])),
('key', self.gf('django.db.models.fields.CharField')(max_length=256)),
('value', self.gf('django.db.models.fields.CharField')(max_length=512)),
))
db.send_create_signal(u'database', ['LanguageString'])
def backwards(self, orm):
# Deleting model 'Language'
db.delete_table(u'database_language')
# Deleting model 'LanguageString'
db.delete_table(u'database_languagestring')
models = {
u'database.block': {
'Meta': {'object_name': 'Block'},
'data_value': ('django.db.models.fields.IntegerField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'})
},
u'database.item': {
'Meta': {'object_name': 'Item'},
'data_value': ('django.db.models.fields.IntegerField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'})
},
'database.jarfile': {
'Meta': {'object_name': 'JarFile'},
'description': ('django.db.models.fields.CharField', [], {'default': "'client'", 'max_length': '256'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Version']"})
},
u'database.language': {
'Meta': {'object_name': 'Language'},
'code': ('django.db.models.fields.CharField', [], {'max_length': '12'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'region': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
u'database.languagestring': {
'Meta': {'object_name': 'LanguageString'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'key': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
'language': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Language']"}),
'value': ('django.db.models.fields.CharField', [], {'max_length': '512'})
},
'database.mod': {
'Meta': {'ordering': "['name']", 'object_name': 'Mod'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
},
u'database.texture': {
'Meta': {'object_name': 'Texture'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'type': ('django.db.models.fields.CharField', [], {'default': "'items'", 'max_length': '16'})
},
'database.version': {
'Meta': {'ordering': "['date']", 'object_name': 'Version'},
'changelog': ('django.db.models.fields.TextField', [], {}),
'date': ('django.db.models.fields.DateField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'mod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Mod']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'snapshot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'release'", 'max_length': '10', 'blank': 'True'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
'version_number': ('django.db.models.fields.CharField', [], {'default': "'0.1.0'", 'max_length': '256'})
}
}
complete_apps = ['database']

89
minecraftcodex/database/migrations/0012_auto.py Normal file
View File

@ -0,0 +1,89 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding index on 'Language', fields ['name']
db.create_index(u'database_language', ['name'])
# Adding index on 'LanguageString', fields ['key']
db.create_index(u'database_languagestring', ['key'])
def backwards(self, orm):
# Removing index on 'LanguageString', fields ['key']
db.delete_index(u'database_languagestring', ['key'])
# Removing index on 'Language', fields ['name']
db.delete_index(u'database_language', ['name'])
models = {
u'database.block': {
'Meta': {'object_name': 'Block'},
'data_value': ('django.db.models.fields.IntegerField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'})
},
u'database.item': {
'Meta': {'object_name': 'Item'},
'data_value': ('django.db.models.fields.IntegerField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'internal_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'main_texture': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Texture']", 'null': 'True'})
},
'database.jarfile': {
'Meta': {'object_name': 'JarFile'},
'description': ('django.db.models.fields.CharField', [], {'default': "'client'", 'max_length': '256'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Version']"})
},
u'database.language': {
'Meta': {'object_name': 'Language'},
'code': ('django.db.models.fields.CharField', [], {'max_length': '12'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'db_index': 'True'}),
'region': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
u'database.languagestring': {
'Meta': {'object_name': 'LanguageString'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'key': ('django.db.models.fields.CharField', [], {'max_length': '256', 'db_index': 'True'}),
'language': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['database.Language']"}),
'value': ('django.db.models.fields.CharField', [], {'max_length': '512'})
},
'database.mod': {
'Meta': {'ordering': "['name']", 'object_name': 'Mod'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
},
u'database.texture': {
'Meta': {'object_name': 'Texture'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'type': ('django.db.models.fields.CharField', [], {'default': "'items'", 'max_length': '16'})
},
'database.version': {
'Meta': {'ordering': "['date']", 'object_name': 'Version'},
'changelog': ('django.db.models.fields.TextField', [], {}),
'date': ('django.db.models.fields.DateField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'mod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['database.Mod']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'snapshot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'release'", 'max_length': '10', 'blank': 'True'}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
'version_number': ('django.db.models.fields.CharField', [], {'default': "'0.1.0'", 'max_length': '256'})
}
}
complete_apps = ['database']

60
minecraftcodex/database/models.py
View File

@ -158,6 +158,18 @@ class Item(models.Model):
main_texture = models.ForeignKey('Texture', null=True)
data_value = models.IntegerField()
def name(self):
result = self.internal_name
try:
string = LanguageString.objects.get(
language=14,
key='item.%s.name' % self.internal_name
)
result = string.value
except:
pass
return result
class ItemAdmin(admin.ModelAdmin):
list_display = ('internal_name', 'data_value', 'main_texture_html')
@ -185,6 +197,18 @@ class Block(models.Model):
main_texture = models.ForeignKey('Texture', null=True)
data_value = models.IntegerField()
def name(self):
result = self.internal_name
try:
string = LanguageString.objects.get(
language=14,
key='tile.%s.name' % self.internal_name
)
result = string.value
except:
pass
return result
class BlockAdmin(admin.ModelAdmin):
list_display = ('internal_name', 'data_value', 'main_texture_html')
@ -202,3 +226,39 @@ class BlockAdmin(admin.ModelAdmin):
main_texture_html.allow_tags = True
admin.site.register(Block, BlockAdmin)
###
# LANGUAGES
###
class Language(models.Model):
name = models.CharField(max_length=64, db_index=True)
region = models.CharField(max_length=32)
code = models.CharField(max_length=12)
def __unicode__(self):
return "%s (%s)" % (self.name, self.region)
class LanguageAdmin(admin.ModelAdmin):
list_display = ('name', 'region', )
list_display_links = ('name', )
search_fields = ('name', )
admin.site.register(Language, LanguageAdmin)
class LanguageString(models.Model):
language = models.ForeignKey('Language', db_index=True)
key = models.CharField(max_length=256, db_index=True)
value = models.CharField(max_length=512)
class LanguageStringAdmin(admin.ModelAdmin):
list_display = ('language', 'key', 'value', )
list_display_links = ('language', 'key', )
list_filter = ('language', )
search_fields = ('key', 'value', )
admin.site.register(LanguageString, LanguageStringAdmin)

2
minecraftcodex/database/templates/blocks.html
View File

@ -25,7 +25,7 @@
</div>
</td>
<td>
{{ item.internal_name }}
{{ item.name() }}
</td>
</tr>
{% endfor %}

2
minecraftcodex/database/templates/items.html
View File

@ -25,7 +25,7 @@
</div>
</td>
<td>
{{ item.internal_name }}
{{ item.name() }}
</td>
</tr>
{% endfor %}

12
utils/miner/conf.py
View File

@ -1,5 +1,6 @@
DEBUG = False
# Save to database?
SAVE = True
###
@ -24,6 +25,17 @@ BLOCKS_FILES = []
BLOCKS_JAVA_KEYWORDS = ['stonebrick']
BLOCKS_PATTERN = "new (?P<code>[a-z]{1,3}\((?P<id>[1-9]{1,3}).*\"(?P<name>\w+)\"\))"
###
# LANGUAGES
###
LANGUAGES_PATH = 'jarfile/lang'
LANGUAGES_MASTER_KEYS = [
'language.name',
'language.region',
'language.code',
]
###
# BLACKLIST
###

144
utils/miner/languages.py Normal file
View File

@ -0,0 +1,144 @@
#!/usr/bin/env python
# General libs
import re
import json
import os
import sys
# Tool libs
from utils import run, sanitize
import conf
from objects import GameLanguage
print("=> Phase: languages")
if conf.SAVE:
sys.path.append('../../minecraftcodex')
os.environ['DJANGO_SETTINGS_MODULE'] = 'local_settings'
from database.models import Language, LanguageString
###
# GLOBALS
###
STRINGS = []
LANGUAGES = []
LANGUAGES_STR = []
###
# LOOK FOR CORRECT JAVA FILES
###
print(" => Looking for languages files...")
directory_list = os.listdir(conf.LANGUAGES_PATH)
print(" Found %d file(s)." % len(directory_list))
###
# GET LANGUAGES
###
try:
OLD_STRINGS = json.loads(open('strings.json').read())
except:
OLD_STRINGS = []
try:
OLD_LANGUAGES = json.loads(open('languages.json').read())
except:
OLD_LANGUAGES = []
print(" => Mining languages...")
for item in directory_list:
if '.lang' in item:
if conf.DEBUG:
print(" Now %s " % item)
language = open('%s/%s' % (conf.LANGUAGES_PATH, item))
language_obj = GameLanguage()
for line in language.readlines():
line = line.strip()
if line and 'X-Generator' not in line:
key, value = line.split('=', 1)
if key in conf.LANGUAGES_MASTER_KEYS:
# Language object
setattr(language_obj, key.split('.')[1], value)
else:
# Language String object
language_obj.add_string(key, value)
# Store for comparision
if language_obj.code == 'en_US':
if key not in STRINGS:
STRINGS.append(key)
LANGUAGES.append(language_obj)
if conf.SAVE:
for item in LANGUAGES:
try:
obj = Language.objects.get(
name=item.name,
region=item.region,
code=item.code
)
except Language.DoesNotExist:
obj = Language(
name=item.name,
region=item.region,
code=item.code
)
obj.save()
for key in item.strings.keys():
value = item.strings[key]
try:
string_obj = LanguageString.objects.get(
language=obj,
key=key
)
if string_obj.value != value:
string_obj.value = value
string_obj.save()
except LanguageString.DoesNotExist:
string_obj = LanguageString(
language=obj,
key=key,
value=value
)
string_obj.save()
print(" => Summary")
# LANGUAGES
[LANGUAGES_STR.append(x.name) for x in LANGUAGES]
new_languages = len(LANGUAGES_STR) - len(OLD_LANGUAGES)
print(" Found %d languages (%d new)." % (len(LANGUAGES_STR), new_languages))
if len(LANGUAGES_STR) != len(OLD_LANGUAGES):
print(" Comparision:")
for string in LANGUAGES_STR:
if string not in OLD_LANGUAGES:
print(" + %s" % string)
for string in OLD_LANGUAGES:
if string not in LANGUAGES_STR:
print(" - %s" % string)
olditems = open('languages.json', 'w')
olditems.write(json.dumps(LANGUAGES_STR))
olditems.close()
# STRINGS
new_strings = len(STRINGS) - len(OLD_STRINGS)
print(" Found %d strings (%d new) -based on en_US-." % (len(STRINGS), new_strings))
if len(STRINGS) != len(OLD_STRINGS):
print(" Comparision:")
for string in STRINGS:
if string not in OLD_STRINGS:
print(" + %s" % string)
for string in OLD_STRINGS:
if string not in STRINGS:
print(" - %s" % string)
olditems = open('strings.json', 'w')
olditems.write(json.dumps(STRINGS))
olditems.close()

21
utils/miner/objects.py
View File

@ -59,3 +59,24 @@ class GameTexture(object):
def parse_name(self, name):
return name.split('.')[0]
###
# LANGUAGES
###
class GameLanguage(object):
def __init__(self, name=None, region=None, code=None):
self.name = name
self.region = region
self.code = code
self.strings = {}
def __str__(self):
return "<Language (%s: '%s')>" % (
self.code,
self.name
)
def add_string(self, key, value):
if key not in self.strings:
self.strings[key] = value

2
utils/miner/run.sh
View File

@ -7,3 +7,5 @@ python textures.py
python items.py
python blocks.py
python languages.py