From 13caccf79f61ada01eb8907a47fa89e64aad0c87 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 21:36:29 +0300 Subject: [PATCH 01/13] migration accounts --- migrate2v2_dump.py | 64 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index dae6bfe..525a69d 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -1,37 +1,60 @@ #!/usr/bin/env python import os from json import dump + +from bitfield import BitHandler, BitField from django import setup +from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import ImageField class BatchSaveStreamList(list): - def __init__(self, model_class, model_name, *args, **kwargs): + def __init__(self, model_class, model_name, except_fields=None, list_map=None, *args, **kwargs): super().__init__(*args, **kwargs) self._model_class = model_class self._model_name = model_name + self._except_fields = (except_fields or []) + ['id'] + self._list_map = list_map or {} - def __iter__(self): - def fields(obj): - return {ob.name: getattr(obj, ob.name) or None for ob in obj._meta.concrete_fields if ob.name != 'id'} + def _fields(self, obj): + return {ob.name: self._list_map_fn(obj, ob) for ob in obj._meta.concrete_fields if + ob.name not in self._except_fields} + def __iter__(self): for d in self._model_class.objects.all().iterator(): yield { "model": self._model_name, "pk": d.pk, - "fields": fields(d) + "fields": self._fields(d) } + def _list_map_fn(self, obj, field): + if field.is_relation: + # fl = getattr(obj, field.name) + val = getattr(obj, field.attname) + return val + elif field.name in self._list_map.keys(): + val = getattr(obj, field.name) + return self._list_map.get(val) + elif isinstance(field, BitField): + val = getattr(obj, field.name) + # val is instance of BitHandler + return int(val) + elif isinstance(field, ImageField): + val = getattr(obj, field.name) + if val._file: + return val.url + else: + return getattr(obj, field.name) or None + def __len__(self): return 1 -def batch_save(fname, model_class, model_name): - sa = BatchSaveStreamList( - model_class=model_class, - model_name=model_name - ) +def batch_save(fname, *args, **kwargs): + sa = BatchSaveStreamList(*args, **kwargs) with open(fname, 'w') as f: - dump(sa, f, ensure_ascii=False, indent=2) + dump(sa, f, ensure_ascii=False, indent=2, cls=DjangoJSONEncoder) # --------------------- @@ -43,12 +66,27 @@ def dump_groups(): def dump_accounts(): - from accounts_app.models import UserProfile, BaseAccount + from accounts_app.models import UserProfile, BaseAccount, UserProfileLog batch_save('accounts_baseaccount.json', BaseAccount, 'profiles.baseaccount') batch_save('accounts_userprofile.json', UserProfile, 'profiles.userprofile') + do_type_map = { + 'cusr': 1, + 'dusr': 2, + 'cdev': 3, + 'ddev': 4, + 'cnas': 5, + 'dnas': 6, + 'csrv': 7, + 'dsrv': 8 + } + batch_save('accounts_userprofilelog.json', UserProfileLog, 'profiles.userprofilelog', + except_fields=['meta_info'], + list_map={ + 'do_type': do_type_map + }) if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_groups() + dump_accounts() From 08869ed4f1ac6190e5a3f6ec3439ce4cc7e010ba Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 21:42:18 +0300 Subject: [PATCH 02/13] add migration messenger --- migrate2v2_dump.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index 525a69d..fc59952 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -2,7 +2,7 @@ import os from json import dump -from bitfield import BitHandler, BitField +from bitfield import BitField from django import setup from django.core.serializers.json import DjangoJSONEncoder from django.db.models import ImageField @@ -86,7 +86,15 @@ def dump_accounts(): }) +def dump_messenger(): + from messenger.models import Messenger, ViberMessenger, ViberMessage, ViberSubscriber + batch_save("messenger.json", Messenger, 'messenger.messenger') + batch_save("ViberMessenger.json", ViberMessenger, 'messenger.vibermessenger') + batch_save("ViberMessage.json", ViberMessage, 'messenger.vibermessage') + batch_save("ViberSubscriber.json", ViberSubscriber, 'messenger.vibersubscriber') + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_accounts() + dump_messenger() From 27e6a8947607f7340428d64735be0986b16c34f4 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 21:48:01 +0300 Subject: [PATCH 03/13] refactoring names --- migrate2v2_dump.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index fc59952..afabb2a 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -9,15 +9,15 @@ from django.db.models import ImageField class BatchSaveStreamList(list): - def __init__(self, model_class, model_name, except_fields=None, list_map=None, *args, **kwargs): + def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, *args, **kwargs): super().__init__(*args, **kwargs) self._model_class = model_class self._model_name = model_name self._except_fields = (except_fields or []) + ['id'] - self._list_map = list_map or {} + self._choice_list_map = choice_list_map or {} def _fields(self, obj): - return {ob.name: self._list_map_fn(obj, ob) for ob in obj._meta.concrete_fields if + return {ob.name: self._field_val(obj, ob) for ob in obj._meta.concrete_fields if ob.name not in self._except_fields} def __iter__(self): @@ -28,22 +28,30 @@ class BatchSaveStreamList(list): "fields": self._fields(d) } - def _list_map_fn(self, obj, field): + def _field_val(self, obj, field): + # related fields if field.is_relation: - # fl = getattr(obj, field.name) val = getattr(obj, field.attname) return val - elif field.name in self._list_map.keys(): + + # choice fields + elif field.name in self._choice_list_map.keys(): val = getattr(obj, field.name) - return self._list_map.get(val) + return self._choice_list_map.get(val) + + # bit fields elif isinstance(field, BitField): val = getattr(obj, field.name) # val is instance of BitHandler return int(val) + + # image fields elif isinstance(field, ImageField): val = getattr(obj, field.name) if val._file: return val.url + + # all other simple fields else: return getattr(obj, field.name) or None @@ -94,6 +102,11 @@ def dump_messenger(): batch_save("ViberSubscriber.json", ViberSubscriber, 'messenger.vibersubscriber') +def dump_services(): + from tariff_app.models import Tariff + batch_save("services.json", Tariff, 'services.service') + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() From 3fa1fe808eef5ddec8295f4d3c40f77d2aa13517 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 22:20:41 +0300 Subject: [PATCH 04/13] add migrate services --- migrate2v2_dump.py | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index afabb2a..2619663 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -9,15 +9,21 @@ from django.db.models import ImageField class BatchSaveStreamList(list): - def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, *args, **kwargs): + def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, **kwargs): super().__init__(*args, **kwargs) self._model_class = model_class self._model_name = model_name self._except_fields = (except_fields or []) + ['id'] self._choice_list_map = choice_list_map or {} + self._field_name_map = field_name_map or {} + + def _map_field_name(self, name): + if name in self._field_name_map: + return self._field_name_map.get(name) + return name def _fields(self, obj): - return {ob.name: self._field_val(obj, ob) for ob in obj._meta.concrete_fields if + return {self._map_field_name(ob.name): self._field_val(obj, ob) for ob in obj._meta.concrete_fields if ob.name not in self._except_fields} def __iter__(self): @@ -37,7 +43,7 @@ class BatchSaveStreamList(list): # choice fields elif field.name in self._choice_list_map.keys(): val = getattr(obj, field.name) - return self._choice_list_map.get(val) + return self._choice_list_map[field.name].get(val) # bit fields elif isinstance(field, BitField): @@ -53,7 +59,10 @@ class BatchSaveStreamList(list): # all other simple fields else: - return getattr(obj, field.name) or None + v = getattr(obj, field.name) + if isinstance(v, bool): + return v + return v or None def __len__(self): return 1 @@ -89,7 +98,7 @@ def dump_accounts(): } batch_save('accounts_userprofilelog.json', UserProfileLog, 'profiles.userprofilelog', except_fields=['meta_info'], - list_map={ + choice_list_map={ 'do_type': do_type_map }) @@ -103,11 +112,28 @@ def dump_messenger(): def dump_services(): - from tariff_app.models import Tariff - batch_save("services.json", Tariff, 'services.service') + from tariff_app.models import Tariff, PeriodicPay + batch_save("services.json", Tariff, 'services.service', field_name_map={ + 'speedIn': 'speed_in', + 'speedOut': 'speed_out', + 'amount': 'cost' + }, choice_list_map={ + 'calc_type': { + 'Df': 0, + 'Dp': 1, + 'Cp': 2, + 'Dl': 3 + } + }) + batch_save("services_periodicpay.json", PeriodicPay, 'services.periodicpay', choice_list_map={ + 'calc_type': { + 'df': 0, + 'cs': 1 + } + }) if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_messenger() + dump_services() From 0e097b9224d29ed16a0a40b0169a1965c40020e6 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 22:28:09 +0300 Subject: [PATCH 05/13] add migrate gateways --- migrate2v2_dump.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index 2619663..5037998 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -133,7 +133,19 @@ def dump_services(): }) +def dump_gateways(): + from gw_app.models import NASModel + batch_save("gateways.json", NASModel, 'gateways.gateway', field_name_map={ + 'nas_type': 'gw_type', + 'default': 'is_default' + }, choice_list_map={ + 'nas_type': { + 'mktk': 0 + } + }) + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_services() + dump_gateways() From 560f36dc5fe12f09a27c457b3010925f4a9e81e9 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 23:02:31 +0300 Subject: [PATCH 06/13] add migrate devices --- migrate2v2_dump.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index 5037998..e99a178 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -7,6 +7,8 @@ from django import setup from django.core.serializers.json import DjangoJSONEncoder from django.db.models import ImageField +from djing.fields import MACAddressField + class BatchSaveStreamList(list): def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, **kwargs): @@ -57,6 +59,11 @@ class BatchSaveStreamList(list): if val._file: return val.url + # mac address validated by netaddr.EUI + elif isinstance(field, MACAddressField): + val = getattr(obj, field.name) + return str(val) + # all other simple fields else: v = getattr(obj, field.name) @@ -145,7 +152,28 @@ def dump_gateways(): }) +def dump_devices(): + from devapp.models import Device, Port + batch_save("devices.json", Device, 'devices.device', field_name_map={ + 'devtype': 'dev_type' + }, choice_list_map={ + 'devtype': { + 'Dl': 1, 'Pn': 2, + 'On': 3, 'Ex': 4, + 'Zt': 5, 'Zo': 6, + 'Z6': 7, 'Hw': 8 + }, + 'status': { + 'und': 0, + 'up': 1, + 'unr': 2, + 'dwn': 3 + } + }) + batch_save('devices_port.json', Port, 'devices.port') + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_gateways() + dump_devices() From 4f5e168aa86219794a3e60e9e04b90065a814bb6 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 23:27:58 +0300 Subject: [PATCH 07/13] add migrate customers and logging progress --- migrate2v2_dump.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index e99a178..2a66bca 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -18,6 +18,7 @@ class BatchSaveStreamList(list): self._except_fields = (except_fields or []) + ['id'] self._choice_list_map = choice_list_map or {} self._field_name_map = field_name_map or {} + print(model_name, end='\t'*3) def _map_field_name(self, name): if name in self._field_name_map: @@ -77,6 +78,7 @@ class BatchSaveStreamList(list): def batch_save(fname, *args, **kwargs): sa = BatchSaveStreamList(*args, **kwargs) + print(fname) with open(fname, 'w') as f: dump(sa, f, ensure_ascii=False, indent=2, cls=DjangoJSONEncoder) @@ -173,7 +175,46 @@ def dump_devices(): batch_save('devices_port.json', Port, 'devices.port') +def dump_customers(): + from abonapp.models import ( + Abon, AbonLog, AbonTariff, AbonStreet, + PassportInfo, InvoiceForPayment, AbonRawPassword, + AdditionalTelephone, PeriodicPayForId + ) + batch_save('customers.json', Abon, 'customers.customer', field_name_map={ + 'current_tariff': 'current_service', + 'ballance': 'balance', + 'nas': 'gateway', + 'autoconnect_service': 'auto_renewal_service', + 'last_connected_tariff': 'last_connected_service' + }) + batch_save('customers_log.json', AbonLog, 'customers.customerlog', field_name_map={ + 'abon': 'customer', + 'amount': 'cost' + }) + batch_save('customers_service.json', AbonTariff, 'customers.customerservice', field_name_map={ + 'tariff': 'service', + 'time_start': 'start_time' + }) + batch_save('customers_street.json', AbonStreet, 'customers.customerstreet') + batch_save('customers_passport.json', PassportInfo, 'customers.passportinfo', field_name_map={ + 'abon': 'customer' + }) + batch_save('customers_inv.json', InvoiceForPayment, 'customers.invoiceforpayment', field_name_map={ + 'abon': 'customer', + 'amount': 'cost' + }) + batch_save('customers_passw.json', AbonRawPassword, 'customers.customerrawpassword', field_name_map={ + 'account': 'customer' + }) + batch_save('customers_tels.json', AdditionalTelephone, 'customers.additionaltelephone', field_name_map={ + 'abon': 'customer' + }) + batch_save('customers_tels.json', PeriodicPayForId, 'customers.periodicpayforid') + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_devices() + dump_accounts() + dump_customers() From 4dff881bd46ea4f752f62b651dbe94b143e08dfb Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Sun, 28 Jul 2019 23:59:47 +0300 Subject: [PATCH 08/13] add migrate networks --- migrate2v2_dump.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index 2a66bca..b26c000 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -5,7 +5,7 @@ from json import dump from bitfield import BitField from django import setup from django.core.serializers.json import DjangoJSONEncoder -from django.db.models import ImageField +from django.db.models import ImageField, ManyToManyField from djing.fields import MACAddressField @@ -26,7 +26,8 @@ class BatchSaveStreamList(list): return name def _fields(self, obj): - return {self._map_field_name(ob.name): self._field_val(obj, ob) for ob in obj._meta.concrete_fields if + fls = obj._meta.get_fields() + return {self._map_field_name(ob.name): self._field_val(obj, ob) for ob in fls if ob.name not in self._except_fields} def __iter__(self): @@ -38,13 +39,8 @@ class BatchSaveStreamList(list): } def _field_val(self, obj, field): - # related fields - if field.is_relation: - val = getattr(obj, field.attname) - return val - # choice fields - elif field.name in self._choice_list_map.keys(): + if field.name in self._choice_list_map.keys(): val = getattr(obj, field.name) return self._choice_list_map[field.name].get(val) @@ -65,6 +61,14 @@ class BatchSaveStreamList(list): val = getattr(obj, field.name) return str(val) + # related fields + if field.is_relation: + val = getattr(obj, field.attname) + if isinstance(field, ManyToManyField): + s = val.only('pk').values_list('pk', flat=True) + return tuple(s) + return val + # all other simple fields else: v = getattr(obj, field.name) @@ -213,8 +217,20 @@ def dump_customers(): batch_save('customers_tels.json', PeriodicPayForId, 'customers.periodicpayforid') +def dump_networks(): + from ip_pool.models import NetworkModel + batch_save('nets.json', NetworkModel, 'networks.networkmodel', choice_list_map={ + 'kind': { + 'inet': 1, + 'guest': 2, + 'trust': 3, + 'device': 4, + 'admin': 5 + } + }) + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_accounts() - dump_customers() + dump_networks() From 87d48099892887924af5ba7c6cd704d1a2014558 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Mon, 29 Jul 2019 00:14:53 +0300 Subject: [PATCH 09/13] add migrate tasks --- migrate2v2_dump.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index b26c000..331d918 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -230,7 +230,41 @@ def dump_networks(): }) +def dump_tasks(): + from taskapp.models import Task, ExtraComment, ChangeLog + batch_save('task.json', Task, 'tasks.task', field_name_map={ + 'abon': 'customer' + }, except_fields=['attachment'], choice_list_map={ + 'priority': { + 'E': 0, + 'C': 1, + 'A': 2 + }, + 'state': { + 'S': 0, + 'C': 1, + 'F': 2 + }, + 'mode': { + 'na', 0, 'ic', 1, + 'yt', 2, 'rc', 3, + 'ls', 4, 'cf', 5, + 'cn', 6, 'pf', 7, + 'cr', 8, 'co', 9, + 'fc', 10, 'ni', 11, + 'ot', 12 + } + }) + batch_save('task_comments.json', ExtraComment, 'tasks.extracomment') + batch_save('task_log.json', ChangeLog, 'tasks.changelog', choice_list_map={ + 'act_type': { + 'e': 1, 'c': 2, + 'd': 3, 'f': 4, 'b': 5 + } + }) + + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_networks() + dump_tasks() From 32170d584162cac5d3f2af2ce286a0520f93dd7a Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Mon, 29 Jul 2019 00:31:27 +0300 Subject: [PATCH 10/13] add migrate tasks --- migrate2v2_dump.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index 331d918..d469db4 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -5,20 +5,21 @@ from json import dump from bitfield import BitField from django import setup from django.core.serializers.json import DjangoJSONEncoder -from django.db.models import ImageField, ManyToManyField +from django.db.models import ImageField, ManyToManyField, ManyToOneRel from djing.fields import MACAddressField class BatchSaveStreamList(list): - def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, **kwargs): + def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, + **kwargs): super().__init__(*args, **kwargs) self._model_class = model_class self._model_name = model_name self._except_fields = (except_fields or []) + ['id'] self._choice_list_map = choice_list_map or {} self._field_name_map = field_name_map or {} - print(model_name, end='\t'*3) + print(model_name, end='\t' * 3) def _map_field_name(self, name): if name in self._field_name_map: @@ -63,6 +64,8 @@ class BatchSaveStreamList(list): # related fields if field.is_relation: + if isinstance(field, ManyToOneRel): + return getattr(obj, field.field_name) val = getattr(obj, field.attname) if isinstance(field, ManyToManyField): s = val.only('pk').values_list('pk', flat=True) @@ -246,13 +249,13 @@ def dump_tasks(): 'F': 2 }, 'mode': { - 'na', 0, 'ic', 1, - 'yt', 2, 'rc', 3, - 'ls', 4, 'cf', 5, - 'cn', 6, 'pf', 7, - 'cr', 8, 'co', 9, - 'fc', 10, 'ni', 11, - 'ot', 12 + 'na': 0, 'ic': 1, + 'yt': 2, 'rc': 3, + 'ls': 4, 'cf': 5, + 'cn': 6, 'pf': 7, + 'cr': 8, 'co': 9, + 'fc': 10, 'ni': 11, + 'ot': 12 } }) batch_save('task_comments.json', ExtraComment, 'tasks.extracomment') From 91300d9de61888f8462de4c239f56add7bf80041 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Mon, 29 Jul 2019 01:06:06 +0300 Subject: [PATCH 11/13] fix --- migrate2v2_dump.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index d469db4..a781674 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -29,7 +29,7 @@ class BatchSaveStreamList(list): def _fields(self, obj): fls = obj._meta.get_fields() return {self._map_field_name(ob.name): self._field_val(obj, ob) for ob in fls if - ob.name not in self._except_fields} + ob.name not in self._except_fields and ob.concrete} def __iter__(self): for d in self._model_class.objects.all().iterator(): @@ -54,8 +54,7 @@ class BatchSaveStreamList(list): # image fields elif isinstance(field, ImageField): val = getattr(obj, field.name) - if val._file: - return val.url + return getattr(val, 'url') if val else None # mac address validated by netaddr.EUI elif isinstance(field, MACAddressField): @@ -103,14 +102,10 @@ def dump_accounts(): batch_save('accounts_baseaccount.json', BaseAccount, 'profiles.baseaccount') batch_save('accounts_userprofile.json', UserProfile, 'profiles.userprofile') do_type_map = { - 'cusr': 1, - 'dusr': 2, - 'cdev': 3, - 'ddev': 4, - 'cnas': 5, - 'dnas': 6, - 'csrv': 7, - 'dsrv': 8 + 'cusr': 1, 'dusr': 2, + 'cdev': 3, 'ddev': 4, + 'cnas': 5, 'dnas': 6, + 'csrv': 7, 'dsrv': 8 } batch_save('accounts_userprofilelog.json', UserProfileLog, 'profiles.userprofilelog', except_fields=['meta_info'], @@ -267,7 +262,15 @@ def dump_tasks(): }) +all_migrs = ( + dump_groups, dump_accounts, dump_messenger, + dump_services, dump_gateways, dump_devices, + dump_customers, dump_networks, dump_tasks +) + if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djing.settings') setup() - dump_tasks() + # dump_accounts() + for migr in all_migrs: + migr() From 6449e0999a65e220f9b632cdb9bb84e25b286b92 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Mon, 29 Jul 2019 02:43:48 +0300 Subject: [PATCH 12/13] fix errs --- migrate2v2_dump.py | 66 ++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/migrate2v2_dump.py b/migrate2v2_dump.py index a781674..0f542a0 100755 --- a/migrate2v2_dump.py +++ b/migrate2v2_dump.py @@ -11,10 +11,10 @@ from djing.fields import MACAddressField class BatchSaveStreamList(list): - def __init__(self, model_class, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, + def __init__(self, model_queryset, model_name, except_fields=None, choice_list_map=None, field_name_map=None, *args, **kwargs): super().__init__(*args, **kwargs) - self._model_class = model_class + self._qs = model_queryset self._model_name = model_name self._except_fields = (except_fields or []) + ['id'] self._choice_list_map = choice_list_map or {} @@ -32,7 +32,7 @@ class BatchSaveStreamList(list): ob.name not in self._except_fields and ob.concrete} def __iter__(self): - for d in self._model_class.objects.all().iterator(): + for d in self._qs.iterator(): yield { "model": self._model_name, "pk": d.pk, @@ -76,7 +76,9 @@ class BatchSaveStreamList(list): v = getattr(obj, field.name) if isinstance(v, bool): return v - return v or None + if field.null: + return v or None + return v def __len__(self): return 1 @@ -94,20 +96,22 @@ def batch_save(fname, *args, **kwargs): def dump_groups(): from group_app.models import Group - batch_save("groups.json", Group, 'groupapp.group') + batch_save("groups.json", Group.objects.all(), 'groupapp.group') def dump_accounts(): from accounts_app.models import UserProfile, BaseAccount, UserProfileLog - batch_save('accounts_baseaccount.json', BaseAccount, 'profiles.baseaccount') - batch_save('accounts_userprofile.json', UserProfile, 'profiles.userprofile') + batch_save('accounts_baseaccount.json', BaseAccount.objects.exclude(username='AnonymousUser'), 'profiles.baseaccount', + except_fields=['groups', 'user_permissions']) + batch_save('accounts_userprofile.json', UserProfile.objects.exclude(username='AnonymousUser'), 'profiles.userprofile', + except_fields=['groups', 'user_permissions']) do_type_map = { 'cusr': 1, 'dusr': 2, 'cdev': 3, 'ddev': 4, 'cnas': 5, 'dnas': 6, 'csrv': 7, 'dsrv': 8 } - batch_save('accounts_userprofilelog.json', UserProfileLog, 'profiles.userprofilelog', + batch_save('accounts_userprofilelog.json', UserProfileLog.objects.all(), 'profiles.userprofilelog', except_fields=['meta_info'], choice_list_map={ 'do_type': do_type_map @@ -116,15 +120,15 @@ def dump_accounts(): def dump_messenger(): from messenger.models import Messenger, ViberMessenger, ViberMessage, ViberSubscriber - batch_save("messenger.json", Messenger, 'messenger.messenger') - batch_save("ViberMessenger.json", ViberMessenger, 'messenger.vibermessenger') - batch_save("ViberMessage.json", ViberMessage, 'messenger.vibermessage') - batch_save("ViberSubscriber.json", ViberSubscriber, 'messenger.vibersubscriber') + batch_save("messenger.json", Messenger.objects.all(), 'messenger.messenger') + batch_save("vibermessenger.json", ViberMessenger.objects.all(), 'messenger.vibermessenger') + batch_save("vibermessage.json", ViberMessage.objects.all(), 'messenger.vibermessage') + batch_save("vibersubscriber.json", ViberSubscriber.objects.all(), 'messenger.vibersubscriber') def dump_services(): from tariff_app.models import Tariff, PeriodicPay - batch_save("services.json", Tariff, 'services.service', field_name_map={ + batch_save("services.json", Tariff.objects.all(), 'services.service', field_name_map={ 'speedIn': 'speed_in', 'speedOut': 'speed_out', 'amount': 'cost' @@ -136,7 +140,7 @@ def dump_services(): 'Dl': 3 } }) - batch_save("services_periodicpay.json", PeriodicPay, 'services.periodicpay', choice_list_map={ + batch_save("services_periodicpay.json", PeriodicPay.objects.all(), 'services.periodicpay', choice_list_map={ 'calc_type': { 'df': 0, 'cs': 1 @@ -146,7 +150,7 @@ def dump_services(): def dump_gateways(): from gw_app.models import NASModel - batch_save("gateways.json", NASModel, 'gateways.gateway', field_name_map={ + batch_save("gateways.json", NASModel.objects.all(), 'gateways.gateway', field_name_map={ 'nas_type': 'gw_type', 'default': 'is_default' }, choice_list_map={ @@ -158,7 +162,7 @@ def dump_gateways(): def dump_devices(): from devapp.models import Device, Port - batch_save("devices.json", Device, 'devices.device', field_name_map={ + batch_save("devices.json", Device.objects.all(), 'devices.device', field_name_map={ 'devtype': 'dev_type' }, choice_list_map={ 'devtype': { @@ -174,7 +178,7 @@ def dump_devices(): 'dwn': 3 } }) - batch_save('devices_port.json', Port, 'devices.port') + batch_save('devices_port.json', Port.objects.all(), 'devices.port') def dump_customers(): @@ -183,41 +187,41 @@ def dump_customers(): PassportInfo, InvoiceForPayment, AbonRawPassword, AdditionalTelephone, PeriodicPayForId ) - batch_save('customers.json', Abon, 'customers.customer', field_name_map={ + batch_save('customer.json', Abon.objects.exclude(username='AnonymousUser'), 'customers.customer', field_name_map={ 'current_tariff': 'current_service', 'ballance': 'balance', 'nas': 'gateway', 'autoconnect_service': 'auto_renewal_service', 'last_connected_tariff': 'last_connected_service' - }) - batch_save('customers_log.json', AbonLog, 'customers.customerlog', field_name_map={ + }, except_fields=['groups', 'user_permissions']) + batch_save('customers_log.json', AbonLog.objects.all(), 'customers.customerlog', field_name_map={ 'abon': 'customer', 'amount': 'cost' }) - batch_save('customers_service.json', AbonTariff, 'customers.customerservice', field_name_map={ + batch_save('customers_service.json', AbonTariff.objects.all(), 'customers.customerservice', field_name_map={ 'tariff': 'service', 'time_start': 'start_time' }) - batch_save('customers_street.json', AbonStreet, 'customers.customerstreet') - batch_save('customers_passport.json', PassportInfo, 'customers.passportinfo', field_name_map={ + batch_save('customers_street.json', AbonStreet.objects.all(), 'customers.customerstreet') + batch_save('customers_passport.json', PassportInfo.objects.all(), 'customers.passportinfo', field_name_map={ 'abon': 'customer' }) - batch_save('customers_inv.json', InvoiceForPayment, 'customers.invoiceforpayment', field_name_map={ + batch_save('customers_inv.json', InvoiceForPayment.objects.all(), 'customers.invoiceforpayment', field_name_map={ 'abon': 'customer', 'amount': 'cost' }) - batch_save('customers_passw.json', AbonRawPassword, 'customers.customerrawpassword', field_name_map={ + batch_save('customers_passw.json', AbonRawPassword.objects.all(), 'customers.customerrawpassword', field_name_map={ 'account': 'customer' }) - batch_save('customers_tels.json', AdditionalTelephone, 'customers.additionaltelephone', field_name_map={ + batch_save('customers_tels.json', AdditionalTelephone.objects.all(), 'customers.additionaltelephone', field_name_map={ 'abon': 'customer' }) - batch_save('customers_tels.json', PeriodicPayForId, 'customers.periodicpayforid') + batch_save('customers_tels.json', PeriodicPayForId.objects.all(), 'customers.periodicpayforid') def dump_networks(): from ip_pool.models import NetworkModel - batch_save('nets.json', NetworkModel, 'networks.networkmodel', choice_list_map={ + batch_save('nets.json', NetworkModel.objects.all(), 'networks.networkmodel', choice_list_map={ 'kind': { 'inet': 1, 'guest': 2, @@ -230,7 +234,7 @@ def dump_networks(): def dump_tasks(): from taskapp.models import Task, ExtraComment, ChangeLog - batch_save('task.json', Task, 'tasks.task', field_name_map={ + batch_save('task.json', Task.objects.all(), 'tasks.task', field_name_map={ 'abon': 'customer' }, except_fields=['attachment'], choice_list_map={ 'priority': { @@ -253,8 +257,8 @@ def dump_tasks(): 'ot': 12 } }) - batch_save('task_comments.json', ExtraComment, 'tasks.extracomment') - batch_save('task_log.json', ChangeLog, 'tasks.changelog', choice_list_map={ + batch_save('task_comments.json', ExtraComment.objects.all(), 'tasks.extracomment') + batch_save('task_log.json', ChangeLog.objects.all(), 'tasks.changelog', choice_list_map={ 'act_type': { 'e': 1, 'c': 2, 'd': 3, 'f': 4, 'b': 5 From 41106d3c4961e67a2bc8a8d6d741ab1ca4541e11 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Mon, 29 Jul 2019 02:44:36 +0300 Subject: [PATCH 13/13] simple refactoring --- tariff_app/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tariff_app/models.py b/tariff_app/models.py index 44f7e0e..7ff47b9 100644 --- a/tariff_app/models.py +++ b/tariff_app/models.py @@ -88,14 +88,14 @@ class PeriodicPay(models.Model): # calc_obj = self._get_calc_object() res = calc_obj.get_next_time_to_pay(self, last_time_payment) - if type(res) is not datetime: + if not isinstance(res, datetime): raise TypeError return res def calc_amount(self): calc_obj = self._get_calc_object() res = calc_obj.calc_amount(self) - if type(res) is not float: + if not isinstance(res, float): raise TypeError return res