From 3548f29277b565196e3b68ad6850c223314a1522 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Wed, 27 Dec 2023 15:49:02 +0100 Subject: [PATCH] add worker-based precalculation of cashbook-values --- __init__.py | 4 + book.py | 65 ++++- cron.py | 18 ++ cron.xml | 15 ++ currency.py | 37 ++- line.py | 25 +- locale/de.po | 40 +++ locale/en.po | 32 +++ message.xml | 3 + mixin.py | 24 +- splitline.py | 4 +- tests/currency.py | 59 ----- tests/line.py | 10 + tests/test_module.py | 4 +- tests/valuestore.py | 610 +++++++++++++++++++++++++++++++++++++++++++ tryton.cfg | 1 + valuestore.py | 207 +++++++++++++++ 17 files changed, 1060 insertions(+), 98 deletions(-) create mode 100644 cron.py create mode 100644 cron.xml delete mode 100644 tests/currency.py create mode 100644 tests/valuestore.py create mode 100644 valuestore.py diff --git a/__init__.py b/__init__.py index 5b2d9f1..7be31cb 100644 --- a/__init__.py +++ b/__init__.py @@ -16,7 +16,9 @@ from .category import Category from .reconciliation import Reconciliation from .cbreport import ReconciliationReport from .currency import CurrencyRate +from .valuestore import ValueStore from .ir import Rule +from .cron import Cron def register(): @@ -34,7 +36,9 @@ def register(): OpenCashBookStart, RunCbReportStart, EnterBookingStart, + ValueStore, Rule, + Cron, module='cashbook', type_='model') Pool.register( ReconciliationReport, diff --git a/book.py b/book.py index 1f6e838..206e839 100644 --- a/book.py +++ b/book.py @@ -4,8 +4,7 @@ # full copyright notices and license terms. from trytond.model import ( - Workflow, ModelView, ModelSQL, fields, Check, - tree, Index) + Workflow, ModelView, ModelSQL, fields, Check, tree, Index) from trytond.pyson import Eval, Or, Bool, Id, Len from trytond.exceptions import UserError from trytond.i18n import gettext @@ -117,6 +116,10 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): 'invisible': STATES2['invisible'], 'required': ~STATES2['invisible'], }, depends=DEPENDS2+['lines']) + + value_store = fields.One2Many( + string='Values', model_name='cashbook.values', field='cashbook', + readonly=True) balance = fields.Function(fields.Numeric( string='Balance', readonly=True, depends=['currency_digits'], @@ -129,7 +132,6 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): help='Balance of all bookings', digits=(16, Eval('currency_digits', 2))), 'get_balance_cashbook', searcher='search_balance') - balance_ref = fields.Function(fields.Numeric( string='Balance (Ref.)', help='Balance in company currency', @@ -383,8 +385,54 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): getattr(tab_line, name), clause[2])) return [('id', 'in', query)] + @classmethod + def valuestore_delete_records(cls, records): + """ delete value-records + """ + ValStore = Pool().get('cashbook.values') + if records: + ValStore.delete_values(records) + + @classmethod + def valuestore_fields(cls): + """ field to update + """ + return ['balance', 'balance_all', 'balance_ref'] + + @classmethod + def valuestore_update_records(cls, records): + """ compute current values of records, + store to global storage + """ + ValStore = Pool().get('cashbook.values') + + if records: + ValStore.update_values( + cls.get_balance_values( + records, + ['balance', 'balance_all', 'balance_ref'])) + @classmethod def get_balance_cashbook(cls, cashbooks, names): + """ get balance of cashbooks + """ + context = Transaction().context + + result = {x: {y.id: Decimal('0.0') for y in cashbooks} for x in names} + + # return computed values if 'date' is in context + query_date = context.get('date', None) + if query_date is not None: + return cls.get_balance_values(cashbooks, names) + + for cashbook in cashbooks: + for value in cashbook.value_store: + if value.field_name in names: + result[value.field_name][cashbook.id] = value.numvalue + return result + + @classmethod + def get_balance_values(cls, cashbooks, names): """ get balance of cashbook """ pool = Pool() @@ -498,6 +546,14 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): """ pass + @classmethod + def create(cls, vlist): + """ update values + """ + records = super(Book, cls).create(vlist) + cls.valuestore_update_records(records) + return records + @classmethod def write(cls, *args): """ deny update if book is not 'open' @@ -506,7 +562,9 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): actions = iter(args) to_write_config = [] + to_update = [] for books, values in zip(actions, actions): + to_update.extend(books) for book in books: # deny btype-->None if lines not empty if 'btype' in values.keys(): @@ -540,6 +598,7 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): if len(to_write_config) > 0: ConfigUser.write(*to_write_config) + cls.valuestore_update_records(to_update) @classmethod def delete(cls, books): diff --git a/cron.py b/cron.py new file mode 100644 index 0000000..36020ae --- /dev/null +++ b/cron.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds.de for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + +from trytond.pool import PoolMeta + + +class Cron(metaclass=PoolMeta): + __name__ = 'ir.cron' + + @classmethod + def __setup__(cls): + super(Cron, cls).__setup__() + cls.method.selection.append( + ('cashbook.values|maintenance_values', "Update Cashbooks")) + +# end Cron diff --git a/cron.xml b/cron.xml new file mode 100644 index 0000000..f1bd33e --- /dev/null +++ b/cron.xml @@ -0,0 +1,15 @@ + + + + + + + cashbook.values|maintenance_values + + hours + + + + diff --git a/currency.py b/currency.py index 190bb39..9b4bf3e 100644 --- a/currency.py +++ b/currency.py @@ -3,7 +3,7 @@ # The COPYRIGHT file at the top level of this repository contains the # full copyright notices and license terms. -from trytond.pool import PoolMeta +from trytond.pool import PoolMeta, Pool class CurrencyRate(metaclass=PoolMeta): @@ -13,22 +13,51 @@ class CurrencyRate(metaclass=PoolMeta): def create(cls, vlist): """ update cache-value """ + pool = Pool() + Cashbook = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + records = super(CurrencyRate, cls).create(vlist) - # TODO: update cashbooks using this rate + + ValueStore.update_books( + ValueStore.get_book_by_books( + Cashbook.search([ + ('currency', 'in', [ + x.currency.id for x in records])]))) return records @classmethod def write(cls, *args): """ update cache-value """ + pool = Pool() + Cashbook = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + + actions = iter(args) + all_rates = [] + for rates, values in zip(actions, actions): + all_rates.extend(rates) + super(CurrencyRate, cls).write(*args) - # TODO: update cashbooks using this rate + + ValueStore.update_books( + ValueStore.get_book_by_books( + Cashbook.search([ + ('currency', 'in', [ + x.currency.id for x in all_rates])]))) @classmethod def delete(cls, records): """ set cache to None """ + pool = Pool() + Cashbook = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + + books = ValueStore.get_book_by_books(Cashbook.search([ + ('currency', 'in', [x.currency.id for x in records])])) super(CurrencyRate, cls).delete(records) - # TODO: update cashbooks using this rate + ValueStore.update_books(books) # end diff --git a/line.py b/line.py index 45dae05..34e4971 100644 --- a/line.py +++ b/line.py @@ -15,7 +15,7 @@ from sql import Literal from sql.functions import DatePart from sql.conditionals import Case from .book import sel_state_book -from .mixin import SecondCurrencyMixin, MemCacheIndexMx +from .mixin import SecondCurrencyMixin from .const import DEF_NONE @@ -49,9 +49,7 @@ STATES = { DEPENDS = ['state', 'state_cashbook'] -class Line( - SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, - ModelView): +class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView): 'Cashbook Line' __name__ = 'cashbook.line' @@ -1037,6 +1035,8 @@ class Line( def create(cls, vlist): """ add debit/credit """ + ValueStore = Pool().get('cashbook.values') + vlist = [x.copy() for x in vlist] for values in vlist: values.update(cls.add_values_from_splitlines(values)) @@ -1056,18 +1056,26 @@ class Line( recname='%(date)s|%(descr)s' % { 'date': date_txt, 'descr': values.get('description', '-')})) - return super(Line, cls).create(vlist) + records = super(Line, cls).create(vlist) + + if records: + ValueStore.update_books(ValueStore.get_book_by_line(records)) + return records @classmethod def write(cls, *args): """ deny update if cashbook.line!='open', add or update debit/credit """ + ValueStore = Pool().get('cashbook.values') + actions = iter(args) to_write = [] + to_update = [] for lines, values in zip(actions, actions): cls.check_permission_write(lines, values) + to_update.extend(lines) for line in lines: if line.reconciliation: # deny state-change to 'edit' if line is @@ -1111,12 +1119,19 @@ class Line( super(Line, cls).write(*to_write) + if to_update: + ValueStore.update_books(ValueStore.get_book_by_line(to_update)) + @classmethod def delete(cls, lines): """ deny delete if book is not 'open' or wf is not 'edit' """ + ValueStore = Pool().get('cashbook.values') + cls.check_permission_delete(lines) + to_update = ValueStore.get_book_by_line(lines) super(Line, cls).delete(lines) + ValueStore.update_books(to_update) # end Line diff --git a/locale/de.po b/locale/de.po index 5d951db..c441d5b 100644 --- a/locale/de.po +++ b/locale/de.po @@ -3,6 +3,14 @@ msgid "" msgstr "Content-Type: text/plain; charset=utf-8\n" +########### +# ir.cron # +########### +msgctxt "selection:ir.cron,method:" +msgid "Update Cashbooks" +msgstr "Kassenbücher Aktualisieren" + + ############## # ir.message # ############## @@ -158,6 +166,10 @@ msgctxt "model:ir.message,text:msg_btype_general" msgid "General" msgstr "Allgemein" +msgctxt "model:ir.message,text:msg_value_exists_in_store" +msgid "The value already exists for the record." +msgstr "Der Wert existiert für den Datensatz bereits. " + ############# # res.group # @@ -598,6 +610,10 @@ msgctxt "field:cashbook.book,booktransf_feature:" msgid "Feature" msgstr "Merkmal" +msgctxt "field:cashbook.book,value_store:" +msgid "Values" +msgstr "Werte" + ################## # cashbook.split # @@ -1645,3 +1661,27 @@ msgstr "Speichern" msgctxt "wizard_button:cashbook.enterbooking,start,savenext_:" msgid "Save & Next" msgstr "Speichern & Weiter" + + +################### +# cashbook.values # +################### +msgctxt "model:cashbook.values,name:" +msgid "Value Store" +msgstr "Wertespeicher" + +msgctxt "field:cashbook.values,resource:" +msgid "Resource" +msgstr "Ressource" + +msgctxt "field:cashbook.values,field_name:" +msgid "Field Name" +msgstr "Feldname" + +msgctxt "field:cashbook.values,numvalue:" +msgid "Value" +msgstr "Wert" + +msgctxt "field:cashbook.values,valuedigits:" +msgid "Digits" +msgstr "Dezimalstellen" diff --git a/locale/en.po b/locale/en.po index f3e6784..0a45ad4 100644 --- a/locale/en.po +++ b/locale/en.po @@ -2,6 +2,10 @@ msgid "" msgstr "Content-Type: text/plain; charset=utf-8\n" +msgctxt "selection:ir.cron,method:" +msgid "Update Cashbooks" +msgstr "Update Cashbooks" + msgctxt "model:ir.message,text:msg_type_short_unique" msgid "The Abbreviation must be unique." msgstr "The Abbreviation must be unique." @@ -154,6 +158,10 @@ msgctxt "model:ir.message,text:msg_btype_general" msgid "General" msgstr "General" +msgctxt "model:ir.message,text:msg_value_exists_in_store" +msgid "The value already exists for the record." +msgstr "The value already exists for the record." + msgctxt "model:res.group,name:group_cashbook" msgid "Cashbook" msgstr "Cashbook" @@ -562,6 +570,10 @@ msgctxt "field:cashbook.book,booktransf_feature:" msgid "Feature" msgstr "Feature" +msgctxt "field:cashbook.book,value_store:" +msgid "Values" +msgstr "Values" + msgctxt "model:cashbook.split,name:" msgid "Split booking line" msgstr "Split booking line" @@ -1550,3 +1562,23 @@ msgctxt "wizard_button:cashbook.enterbooking,start,save_:" msgid "Save" msgstr "Save" +msgctxt "wizard_button:cashbook.enterbooking,start,savenext_:" +msgid "Save & Next" +msgstr "Save & Next" + +msgctxt "model:cashbook.values,name:" +msgid "Value Store" +msgstr "Value Store" + +msgctxt "field:cashbook.values,resource:" +msgid "Resource" +msgstr "Resource" + +msgctxt "field:cashbook.values,field_name:" +msgid "Field Name" +msgstr "Field Name" + +msgctxt "field:cashbook.values,numvalue:" +msgid "Value" +msgstr "Value" + diff --git a/message.xml b/message.xml index ed21703..ff2d735 100644 --- a/message.xml +++ b/message.xml @@ -119,6 +119,9 @@ full copyright notices and license terms. --> General + + The value already exists for the record. + diff --git a/mixin.py b/mixin.py index 363bed5..e261af2 100644 --- a/mixin.py +++ b/mixin.py @@ -3,7 +3,7 @@ # The COPYRIGHT file at the top level of this repository contains the # full copyright notices and license terms. -from trytond.model import fields, Index +from trytond.model import fields from trytond.pyson import Eval, Bool, Or from trytond.pool import Pool from trytond.modules.currency.ir import rate_decimal @@ -196,25 +196,3 @@ class SecondCurrencyMixin: return 2 # end SecondCurrencyMixin - - -class MemCacheIndexMx: - """ add index to 'create_date' + 'write_date' - """ - __slots__ = () - - @classmethod - def __setup__(cls): - super(MemCacheIndexMx, cls).__setup__() - t = cls.__table__() - # add index - cls._sql_indexes.update({ - Index( - t, - (t.write_date, Index.Range())), - Index( - t, - (t.create_date, Index.Range())), - }) - -# end MemCacheIndexMx diff --git a/splitline.py b/splitline.py index 79e78df..6546992 100644 --- a/splitline.py +++ b/splitline.py @@ -11,7 +11,7 @@ from trytond.report import Report from trytond.i18n import gettext from .line import sel_bookingtype, sel_linestate, STATES, DEPENDS from .book import sel_state_book -from .mixin import SecondCurrencyMixin, MemCacheIndexMx +from .mixin import SecondCurrencyMixin sel_linetype = [ @@ -25,7 +25,7 @@ sel_target = [ ] -class SplitLine(SecondCurrencyMixin, MemCacheIndexMx, ModelSQL, ModelView): +class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView): 'Split booking line' __name__ = 'cashbook.split' diff --git a/tests/currency.py b/tests/currency.py deleted file mode 100644 index a63e6ec..0000000 --- a/tests/currency.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of the cashbook-module from m-ds for Tryton. -# The COPYRIGHT file at the top level of this repository contains the -# full copyright notices and license terms. - -from trytond.tests.test_tryton import with_transaction -from trytond.pool import Pool -from datetime import date -from decimal import Decimal - - -class CurrencyTestCase(object): - """ test currency - """ - @with_transaction() - def test_currency_update_cache(self): - """ add/update/del rate of currency, check cache - """ - pool = Pool() - Currency = pool.get('currency.currency') - CurrencyRate = pool.get('currency.currency.rate') - - self.prep_config() - self.prep_company() - - # TODO: check update of cashbook if currency changes - - currency, = Currency.search([('name', '=', 'usd')]) - CurrencyRate.delete(currency.rates) - self.assertEqual(len(currency.rates), 0) - - # add rate - Currency.write(*[ - [currency], - { - 'rates': [('create', [{ - 'date': date(2022, 5, 1), - 'rate': Decimal('1.05'), - }])], - }]) - self.assertEqual(len(currency.rates), 1) - - Currency.write(*[ - [currency], - { - 'rates': [ - ('write', [currency.rates[0].id], { - 'rate': Decimal('1.06'), - })], - }]) - self.assertEqual(len(currency.rates), 1) - - Currency.write(*[ - [currency], - { - 'rates': [('delete', [currency.rates[0].id])], - }]) - -# end CurrencyTestCase diff --git a/tests/line.py b/tests/line.py index 63a6682..2175a11 100644 --- a/tests/line.py +++ b/tests/line.py @@ -291,6 +291,8 @@ class LineTestCase(object): self.assertEqual(books[0].lines[0].reference, None) self.assertEqual(len(books[0].lines[0].references), 1) + self.prep_valstore_run_worker() + self.assertEqual( books[0].lines[0].rec_name, '05/05/2022|to|-10.00 usd|Transfer USD --> ' + @@ -366,6 +368,8 @@ class LineTestCase(object): self.assertEqual(books[0].lines[0].reference, None) self.assertEqual(len(books[0].lines[0].references), 1) + self.prep_valstore_run_worker() + self.assertEqual( books[0].lines[0].rec_name, '05/05/2022|from|10.00 usd|Transfer USD <-- ' + @@ -458,6 +462,8 @@ class LineTestCase(object): self.assertEqual(books[0].lines[0].reference, None) self.assertEqual(len(books[0].lines[0].references), 1) + self.prep_valstore_run_worker() + # 10 CHF --> USD: USD = CHF * 1.05 / 1.04 # 10 CHF = 10.0961538 USD # EUR | USD | CHF @@ -1147,6 +1153,8 @@ class LineTestCase(object): # set line to 'checked', this creates the counterpart Line.wfcheck(list(book.lines)) + self.prep_valstore_run_worker() + self.assertEqual(len(book.lines), 1) self.assertEqual( book.lines[0].rec_name, @@ -1216,6 +1224,8 @@ class LineTestCase(object): # set line to 'checked', this creates the counterpart Line.wfcheck(list(book.lines)) + self.prep_valstore_run_worker() + self.assertEqual(len(book.lines), 1) self.assertEqual( book.lines[0].rec_name, diff --git a/tests/test_module.py b/tests/test_module.py index 87f9d68..f2f5687 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -14,11 +14,11 @@ from .config import ConfigTestCase from .category import CategoryTestCase from .reconciliation import ReconTestCase from .bookingwiz import BookingWizardTestCase -from .currency import CurrencyTestCase +from .valuestore import ValuestoreTestCase class CashbookTestCase( - CurrencyTestCase, + ValuestoreTestCase, BookingWizardTestCase, ReconTestCase, CategoryTestCase, diff --git a/tests/valuestore.py b/tests/valuestore.py new file mode 100644 index 0000000..fed00ab --- /dev/null +++ b/tests/valuestore.py @@ -0,0 +1,610 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + +from datetime import date, timedelta +from decimal import Decimal +from trytond.tests.test_tryton import with_transaction +from trytond.pool import Pool +from trytond.exceptions import UserError +from trytond.transaction import Transaction + + +class ValuestoreTestCase(object): + """ test storage of values + """ + @with_transaction() + def test_valstore_update_currency_rate(self): + """ create cashbook, check update of cashbook on + update of rates of currency + """ + pool = Pool() + Book = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + Currency = pool.get('currency.currency') + Queue = pool.get('ir.queue') + + types = self.prep_type() + company = self.prep_company() + (usd, euro) = self.prep_2nd_currency(company) + self.assertEqual(company.currency.rec_name, 'Euro') + + with Transaction().set_context({ + 'company': company.id, + 'date': date(2022, 5, 20)}): + + self.assertEqual(Queue.search_count([]), 0) + + category = self.prep_category(cattype='in') + party = self.prep_party() + book, = Book.create([{ + 'name': 'Book 1', + 'btype': types.id, + 'company': company.id, + 'currency': usd.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + 'lines': [('create', [{ + 'date': date(2022, 5, 1), + 'description': '10 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('10.0'), + 'party': party.id, + }, { + 'date': date(2022, 5, 10), + 'description': '5 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('5.0'), + 'party': party.id, + }])], + }]) + + # run worker + self.assertEqual(ValueStore.search_count([]), 3) + self.prep_valstore_run_worker() + + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 15.00 usd | Open') + self.assertEqual(book.balance, Decimal('15.0')) + self.assertEqual(book.balance_all, Decimal('15.0')) + self.assertEqual(book.balance_ref, Decimal('14.29')) + self.assertEqual(len(book.value_store), 3) + self.assertEqual( + book.value_store[0].rec_name, + '[Book 1 | 15.00 usd | Open]|balance|15.00|2') + self.assertEqual( + book.value_store[1].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + book.value_store[2].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_ref|14.29|2') + + # add rate to usd + self.assertEqual(Queue.search_count([]), 0) + Currency.write(*[ + [usd], + { + 'rates': [('create', [{ + 'date': date(2022, 5, 6), + 'rate': Decimal('1.08'), + }])], + }]) + self.assertEqual(Queue.search_count([]), 1) + self.prep_valstore_run_worker() + self.assertEqual(Queue.search_count([]), 0) + + # check reference-currency + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 15.00 usd | Open') + self.assertEqual(book.balance, Decimal('15.0')) + self.assertEqual(book.balance_all, Decimal('15.0')) + self.assertEqual(book.balance_ref, Decimal('13.89')) + self.assertEqual(len(book.value_store), 3) + self.assertEqual( + book.value_store[0].rec_name, + '[Book 1 | 15.00 usd | Open]|balance|15.00|2') + self.assertEqual( + book.value_store[1].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + book.value_store[2].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_ref|13.89|2') + + # find rate + self.assertEqual(len(usd.rates), 2) + self.assertEqual(usd.rates[0].date, date(2022, 5, 6)) + self.assertEqual(usd.rates[0].rate, Decimal('1.08')) + + # update rate + self.assertEqual(Queue.search_count([]), 0) + Currency.write(*[ + [usd], + { + 'rates': [( + 'write', + [usd.rates[0]], + {'rate': Decimal('1.12')})], + }]) + self.assertEqual(Queue.search_count([]), 1) + self.prep_valstore_run_worker() + self.assertEqual(Queue.search_count([]), 0) + + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 15.00 usd | Open') + self.assertEqual(book.balance_ref, Decimal('13.39')) + self.assertEqual( + book.value_store[2].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_ref|13.39|2') + + # delete rate + self.assertEqual(Queue.search_count([]), 0) + Currency.write(*[ + [usd], + { + 'rates': [('delete', [usd.rates[0]])], + }]) + self.assertEqual(Queue.search_count([]), 1) + self.prep_valstore_run_worker() + self.assertEqual(Queue.search_count([]), 0) + + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 15.00 usd | Open') + self.assertEqual(book.balance_ref, Decimal('14.29')) + self.assertEqual( + book.value_store[2].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_ref|14.29|2') + + def prep_valstore_run_worker(self): + """ run tasks from queue + """ + Queue = Pool().get('ir.queue') + + tasks = Queue.search([]) + for task in tasks: + task.run() + Queue.delete(tasks) + + @with_transaction() + def test_valstore_update_store_values(self): + """ create cashbook, store value + """ + pool = Pool() + Book = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + Queue = pool.get('ir.queue') + + types = self.prep_type() + company = self.prep_company() + (usd, euro) = self.prep_2nd_currency(company) + self.assertEqual(company.currency.rec_name, 'Euro') + + with Transaction().set_context({'company': company.id}): + + self.assertEqual(Queue.search_count([]), 0) + + category = self.prep_category(cattype='in') + party = self.prep_party() + book, = Book.create([{ + 'name': 'Book 1', + 'btype': types.id, + 'company': company.id, + 'currency': usd.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + 'lines': [('create', [{ + 'date': date(2022, 5, 1), + 'description': '10 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('10.0'), + 'party': party.id, + }, { + 'date': date(2022, 5, 10), + 'description': '5 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('5.0'), + 'party': party.id, + }])], + }]) + + # run worker + self.assertEqual(ValueStore.search_count([]), 3) + self.prep_valstore_run_worker() + + # check values until 2022-05-05 + with Transaction().set_context({ + 'date': date(2022, 5, 5)}): + + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 10.00 usd | Open') + self.assertEqual(book.balance, Decimal('10.0')) + self.assertEqual(book.balance_all, Decimal('15.0')) + self.assertEqual(book.balance_ref, Decimal('14.29')) + self.assertEqual(len(book.value_store), 3) + self.assertEqual( + book.value_store[0].rec_name, + '[Book 1 | 10.00 usd | Open]|balance|15.00|2') + self.assertEqual( + book.value_store[1].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + book.value_store[2].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_ref|14.29|2') + + # values created by book-create, without context + self.assertEqual(ValueStore.search_count([]), 3) + + values = ValueStore.search([], order=[('field_name', 'ASC')]) + self.assertEqual(len(values), 3) + self.assertEqual( + values[0].rec_name, + '[Book 1 | 10.00 usd | Open]|balance|15.00|2') + self.assertEqual( + values[1].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + values[2].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_ref|14.29|2') + + # check write of too much digits + self.assertRaisesRegex( + UserError, + r'The number of digits in the value ' + + r'"' + r"'12.345'" + + r'" for field "Value" in "[Book 1 | 10\.00 usd | ' + + r'Open]|balance|12\.35|2" of "Value Store" exceeds ' + + r'the limit of "2".', + ValueStore.write, + *[ + [values[0]], + { + 'numvalue': Decimal('12.345'), + } + ]) + + # update with context + Book.valuestore_update_records([book]) + + values = ValueStore.search([], order=[('field_name', 'ASC')]) + self.assertEqual(len(values), 3) + + self.assertEqual( + values[0].rec_name, + '[Book 1 | 10.00 usd | Open]|balance|10.00|2') + self.assertEqual( + values[1].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + values[2].rec_name, + '[Book 1 | 10.00 usd | Open]|balance_ref|14.29|2') + + # check values until 2022-05-15 + with Transaction().set_context({ + 'date': date(2022, 5, 15)}): + + book, = Book.search([]) + self.assertEqual(book.rec_name, 'Book 1 | 15.00 usd | Open') + self.assertEqual(book.balance, Decimal('15.0')) + self.assertEqual(book.balance_all, Decimal('15.0')) + self.assertEqual(book.balance_ref, Decimal('14.29')) + + # update values + self.assertEqual(ValueStore.search_count([]), 3) + Book.valuestore_update_records([book]) + + values = ValueStore.search([], order=[('field_name', 'ASC')]) + self.assertEqual(len(values), 3) + + self.assertEqual( + values[0].rec_name, + '[Book 1 | 15.00 usd | Open]|balance|15.00|2') + self.assertEqual( + values[1].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_all|15.00|2') + self.assertEqual( + values[2].rec_name, + '[Book 1 | 15.00 usd | Open]|balance_ref|14.29|2') + + # delete book, should delete values + Book.write(*[ + [book], + {'lines': [('delete', [x.id for x in book.lines])]} + ]) + self.assertEqual(ValueStore.search_count([]), 3) + Book.delete([book]) + self.assertEqual(ValueStore.search_count([]), 0) + + @with_transaction() + def test_valstore_update_store_values_line(self): + """ create cashbooks hierarchical, add lines + check update of parent cashbooks + """ + pool = Pool() + Book = pool.get('cashbook.book') + Line = pool.get('cashbook.line') + ValueStore = pool.get('cashbook.values') + + types = self.prep_type() + company = self.prep_company() + (usd, euro) = self.prep_2nd_currency(company) + self.assertEqual(company.currency.rec_name, 'Euro') + + with Transaction().set_context({'company': company.id}): + category = self.prep_category(cattype='in') + party = self.prep_party() + book, = Book.create([{ + 'name': 'Lev 0', + 'btype': types.id, + 'company': company.id, + 'currency': usd.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + 'lines': [('create', [{ + 'date': date(2022, 5, 1), + 'description': '10 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('10.0'), + 'party': party.id, + }])], + 'childs': [('create', [{ + 'name': 'Lev 1a', + 'btype': types.id, + 'company': company.id, + 'currency': euro.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + }, { + 'name': 'Lev 1b', + 'btype': types.id, + 'company': company.id, + 'currency': euro.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + }])], + }]) + self.assertEqual(book.rec_name, 'Lev 0 | 10.00 usd | Open') + self.assertEqual(len(book.lines), 1) + self.assertEqual( + book.lines[0].rec_name, + '05/01/2022|Rev|10.00 usd|10 US$ [Cat1]') + self.assertEqual(len(book.childs), 2) + self.assertEqual( + book.childs[0].rec_name, 'Lev 0/Lev 1a | 0.00 € | Open') + self.assertEqual(len(book.childs[0].lines), 0) + self.assertEqual( + book.childs[1].rec_name, 'Lev 0/Lev 1b | 0.00 € | Open') + self.assertEqual(len(book.childs[1].lines), 0) + + self.assertEqual(ValueStore.search_count([]), 9) + self.prep_valstore_run_worker() + + values = ValueStore.search( + [], order=[('cashbook', 'ASC'), ('field_name', 'ASC')]) + self.assertEqual(len(values), 9) + + self.assertEqual( + values[0].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance|10.00|2') + self.assertEqual( + values[1].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance_all|10.00|2') + self.assertEqual( + values[2].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance_ref|9.52|2') + + self.assertEqual( + values[3].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[4].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[5].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance_ref|0.00|2') + + self.assertEqual( + values[6].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[7].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[8].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_ref|0.00|2') + + # add bookings + Line.create([{ + 'cashbook': values[0].cashbook.id, # Lev 0 + 'amount': Decimal('2.0'), + 'bookingtype': 'in', + 'category': category.id, + 'date': date(2022, 5, 10), + }, { + 'cashbook': values[3].cashbook.id, # Lev 1a + 'amount': Decimal('3.0'), + 'bookingtype': 'in', + 'category': category.id, + 'date': date(2022, 5, 10), + }]) + + # add 'date' to context, will return computed + # (not stored) values + with Transaction().set_context({'date': date(2022, 5, 10)}): + values = ValueStore.search( + [], order=[('cashbook', 'ASC'), ('field_name', 'ASC')]) + self.assertEqual(len(values), 9) + self.assertEqual( + values[0].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance|10.00|2') + self.assertEqual( + values[1].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance_all|10.00|2') + self.assertEqual( + values[2].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance_ref|9.52|2') + + self.assertEqual( + values[3].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[4].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[5].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance_ref|0.00|2') + + self.assertEqual( + values[6].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[7].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[8].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_ref|0.00|2') + + # before run of workers - w/o 'date' in context + values = ValueStore.search( + [], order=[('cashbook', 'ASC'), ('field_name', 'ASC')]) + self.assertEqual(len(values), 9) + self.assertEqual( + values[0].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance|10.00|2') + self.assertEqual( + values[1].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance_all|10.00|2') + self.assertEqual( + values[2].rec_name, + '[Lev 0 | 10.00 usd | Open]|balance_ref|9.52|2') + + self.assertEqual( + values[3].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[4].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[5].rec_name, + '[Lev 0/Lev 1a | 0.00 € | Open]|balance_ref|0.00|2') + + self.assertEqual( + values[6].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[7].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[8].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_ref|0.00|2') + + self.prep_valstore_run_worker() + + # after run of workers + values = ValueStore.search( + [], order=[('cashbook', 'ASC'), ('field_name', 'ASC')]) + self.assertEqual(len(values), 9) + + self.assertEqual( + values[0].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance|15.15|2') + self.assertEqual( + values[1].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance_all|15.15|2') + self.assertEqual( + values[2].rec_name, + '[Lev 0 | 15.15 usd | Open]|balance_ref|14.43|2') + + self.assertEqual( + values[3].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance|3.00|2') + self.assertEqual( + values[4].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance_all|3.00|2') + self.assertEqual( + values[5].rec_name, + '[Lev 0/Lev 1a | 3.00 € | Open]|balance_ref|3.00|2') + + self.assertEqual( + values[6].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance|0.00|2') + self.assertEqual( + values[7].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_all|0.00|2') + self.assertEqual( + values[8].rec_name, + '[Lev 0/Lev 1b | 0.00 € | Open]|balance_ref|0.00|2') + + @with_transaction() + def test_valstore_maintain_values(self): + """ create cashbook, check maintenance - + update records by cron, delete lost records + """ + pool = Pool() + Book = pool.get('cashbook.book') + ValueStore = pool.get('cashbook.values') + tab_book = Book.__table__() + cursor = Transaction().connection.cursor() + + types = self.prep_type() + company = self.prep_company() + (usd, euro) = self.prep_2nd_currency(company) + self.assertEqual(company.currency.rec_name, 'Euro') + + category = self.prep_category(cattype='in') + party = self.prep_party() + book, = Book.create([{ + 'name': 'Book 1', + 'btype': types.id, + 'company': company.id, + 'currency': usd.id, + 'number_sequ': self.prep_sequence().id, + 'start_date': date(2022, 5, 1), + 'lines': [('create', [{ + 'date': date(2022, 5, 1), + 'description': '10 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('10.0'), + 'party': party.id, + }, { + 'date': date(2022, 5, 10), + 'description': '5 US$', + 'category': category.id, + 'bookingtype': 'in', + 'amount': Decimal('5.0'), + 'party': party.id, + }])], + }]) + + # clean 'numvalue' + # maintenance_values() will restore it + val1, = ValueStore.search([('field_name', '=', 'balance')]) + ValueStore.write(*[[val1], {'numvalue': None}]) + + self.assertTrue(val1.write_date is not None) + self.assertTrue(val1.create_date is not None) + self.assertEqual(val1.numvalue, None) + + # update outdated records + with Transaction().set_context({ + 'maintenance_date': val1.write_date.date() + + timedelta(days=2)}): + ValueStore.maintenance_values() + + val1, = ValueStore.search([('field_name', '=', 'balance')]) + self.assertEqual(val1.numvalue, Decimal('15.0')) + + # delete book + self.assertEqual(Book.search_count([]), 1) + self.assertEqual(ValueStore.search_count([]), 3) + query = tab_book.delete(where=tab_book.id == book.id) + cursor.execute(*query) + self.assertEqual(Book.search_count([]), 0) + self.assertEqual(ValueStore.search_count([]), 0) + +# end ValuestoreTestCase diff --git a/tryton.cfg b/tryton.cfg index a630e41..1da7529 100644 --- a/tryton.cfg +++ b/tryton.cfg @@ -21,3 +21,4 @@ xml: wizard_runreport.xml wizard_booking.xml menu.xml + cron.xml diff --git a/valuestore.py b/valuestore.py new file mode 100644 index 0000000..1a4cabf --- /dev/null +++ b/valuestore.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + + +from sql.functions import CurrentTimestamp, DateTrunc +from sql.aggregate import Count +from sql.conditionals import Coalesce +from trytond.model import ModelSQL, fields, Unique, Index +from trytond.pool import Pool +from trytond.transaction import Transaction +from trytond.pyson import Eval, PYSON, PYSONEncoder, PYSONDecoder +from trytond.model.modelstorage import EvalEnvironment +from trytond.report import Report + + +class ValueStore(ModelSQL): + 'Value Store' + __name__ = 'cashbook.values' + + cashbook = fields.Many2One( + string='Cashbook', required=True, model_name='cashbook.book', + ondelete='CASCADE') + field_name = fields.Char(string='Field Name', required=True) + numvalue = fields.Numeric( + string='Value', digits=(16, Eval('valuedigits', 6)), + depends=['valuedigits']) + valuedigits = fields.Function(fields.Integer( + string='Digits', readonly=True), + 'on_change_with_valuedigits') + + @classmethod + def __setup__(cls): + super(ValueStore, cls).__setup__() + t = cls.__table__() + cls._sql_constraints.extend([ + ('uniqu_field', + Unique(t, t.cashbook, t.field_name), + 'cashbook.msg_value_exists_in_store'), + ]) + cls._sql_indexes.update({ + Index( + t, + (t.field_name, Index.Equality())), + }) + + def get_rec_name(self, name): + """ name, balance, state + """ + return '|'.join([ + '[' + getattr(self.cashbook, 'rec_name', '-') + ']', + self.field_name or '-', + Report.format_number( + self.numvalue, + None, + digits=self.valuedigits or 2) + if self.numvalue is not None else '-', + str(self.valuedigits) if self.valuedigits is not None else '-']) + + @fields.depends('cashbook', 'field_name') + def on_change_with_valuedigits(self, name=None): + """ get digits by field name + """ + Cashbook = Pool().get('cashbook.book') + + if self.cashbook and self.field_name: + fieldobj = Cashbook._fields.get(self.field_name, None) + if fieldobj: + digit = getattr(fieldobj, 'digits', (16, 6))[1] + if isinstance(digit, PYSON): + # execute pyson on linked record + digit = PYSONDecoder( + EvalEnvironment(self.cashbook, Cashbook) + ).decode(PYSONEncoder().encode(digit)) + return digit + return 6 + + @classmethod + def _maintenance_fields(cls): + """ list of model and fieldnames, + to daily update + """ + return ['balance'] + + @classmethod + def maintenance_values(cls): + """ update values by cron + """ + pool = Pool() + Cashbook = pool.get('cashbook.book') + tab_val = cls.__table__() + tab_book = Cashbook.__table__() + cursor = Transaction().connection.cursor() + context = Transaction().context + + # select records older than 'check_dt' + check_dt = context.get('maintenance_date', None) + if not check_dt: + check_dt = DateTrunc('day', CurrentTimestamp()) + + # select records to update + query = tab_val.select( + tab_val.id, + where=tab_val.field_name.in_(cls._maintenance_fields()) & + (DateTrunc('day', Coalesce( + tab_val.write_date, + tab_val.create_date, + '1990-01-01 00:00:00')) < check_dt)) + cursor.execute(*query) + records = [] + for x in cursor.fetchall(): + cb = cls(x[0]).cashbook + if cb not in records: + records.append(cb) + + # add records with missing fields in value-store + num_fields = len(Cashbook.valuestore_fields()) + query = tab_book.join( + tab_val, + condition=tab_book.id == tab_val.cashbook, + type_='LEFT OUTER', + ).select( + tab_book.id, + Count(tab_val.id).as_('num'), + group_by=[tab_book.id], + having=Count(tab_val.id) < num_fields) + cursor.execute(*query) + records.extend([Cashbook(x[0]) for x in cursor.fetchall()]) + + if records: + Cashbook.valuestore_update_records([x for x in records]) + + @classmethod + def get_book_by_line(cls, records): + """ select books above current record to update + records: cashbook.line + """ + Book = Pool().get('cashbook.book') + + to_update = [] + if records: + books = Book.search([ + ('parent', 'parent_of', [x.cashbook.id for x in records]) + ]) + to_update.extend([ + x for x in books + if x not in to_update]) + return to_update + + @classmethod + def get_book_by_books(cls, records): + """ select books above current record to update + records: cashbook.book + """ + Book = Pool().get('cashbook.book') + + to_update = [] + if records: + books = Book.search([ + ('parent', 'parent_of', [x.id for x in records]) + ]) + to_update.extend([ + x for x in books + if x not in to_update]) + return to_update + + @classmethod + def update_books(cls, books): + """ get cashbooks to update, queue it + """ + Book = Pool().get('cashbook.book') + + if books: + Book.__queue__.valuestore_update_records(books) + + @classmethod + def update_values(cls, data): + """ data: {'fieldname': {id1: value, id2: value, ...}, ...} + """ + to_write = [] + to_create = [] + + for name in data.keys(): + for record_id in data[name].keys(): + # get existing record + records = cls.search([ + ('cashbook', '=', record_id), + ('field_name', '=', name)]) + + if records: + for record in records: + to_write.extend([ + [record], + {'numvalue': data[name][record_id]}]) + else: + to_create.append({ + 'cashbook': record_id, + 'field_name': name, + 'numvalue': data[name][record_id]}) + + if to_create: + cls.create(to_create) + if to_write: + cls.write(*to_write) + +# end ValueStore