diff --git a/__init__.py b/__init__.py index f7075c0..20541f8 100644 --- a/__init__.py +++ b/__init__.py @@ -15,11 +15,15 @@ from .configuration import Configuration, UserConfiguration from .category import Category from .reconciliation import Reconciliation from .cbreport import ReconciliationReport +from .currency import CurrencyRate +from .model import MemCache def register(): Pool.register( + MemCache, Configuration, UserConfiguration, + CurrencyRate, Type, Category, Book, diff --git a/book.py b/book.py index 84347c6..70c4e24 100644 --- a/book.py +++ b/book.py @@ -13,9 +13,9 @@ from trytond.report import Report from decimal import Decimal from datetime import date from sql.aggregate import Sum -from sql.conditionals import Case, Coalesce -from sql.functions import CurrentDate -from .model import order_name_hierarchical, sub_ids_hierarchical, AnyInArray +from sql.conditionals import Case +from .model import order_name_hierarchical, sub_ids_hierarchical, \ + AnyInArray, CACHEKEY_CURRENCY STATES = { @@ -48,12 +48,12 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): __name__ = 'cashbook.book' company = fields.Many2One(string='Company', model_name='company.company', - required=True, ondelete="RESTRICT") + required=True, select=True, ondelete="RESTRICT") name = fields.Char(string='Name', required=True, states=STATES, depends=DEPENDS) description = fields.Text(string='Description', states=STATES, depends=DEPENDS) - btype = fields.Many2One(string='Type', + btype = fields.Many2One(string='Type', select=True, help='A cash book with type can contain postings. Without type is a view.', model_name='cashbook.type', ondelete='RESTRICT', states={ @@ -346,12 +346,15 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): Currency = pool.get('currency.currency') Company = pool.get('company.company') IrDate = pool.get('ir.date') + MemCache = pool.get('cashbook.memcache') tab_book = Book2.__table__() tab_comp = Company.__table__() cursor = Transaction().connection.cursor() context = Transaction().context - result = {x:{y.id: Decimal('0.0') for y in cashbooks} for x in names} + result = { + x:{y.id: Decimal('0.0') for y in cashbooks} + for x in ['balance', 'balance_all', 'balance_ref']} # deny invalid date in context query_date = context.get('date', IrDate.today()) @@ -361,6 +364,28 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): except : query_date = IrDate.today() + cache_keys = { + x.id: MemCache.get_key_by_record( + name = 'get_balance_cashbook', + record = x, + query = [{ + 'model': 'cashbook.line', + 'query': [('cashbook.parent', 'child_of', [x.id])], + }, { + 'model': 'currency.currency.rate', + 'query': [('currency.id', '=', x.currency.id)], + 'cachekey': CACHEKEY_CURRENCY % x.currency.id, + }, ], + addkeys = [query_date.isoformat()]) + for x in cashbooks + } + + # read from cache + (todo_cashbook, result) = MemCache.read_from_cache( + cashbooks, cache_keys, names, result) + if len(todo_cashbook) == 0: + return result + # query balances of cashbooks and sub-cashbooks with Transaction().set_context({ 'date': query_date, @@ -381,26 +406,20 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView): Sum(tab_line.balance).as_('balance'), Sum(tab_line.balance_all).as_('balance_all'), group_by=[tab_book.id, tab_line.currency, tab_comp.currency], - where=tab_book.id.in_([x.id for x in cashbooks]), + where=tab_book.id.in_([x.id for x in todo_cashbook]), ) cursor.execute(*query) records = cursor.fetchall() for record in records: - values = { - 'balance': Currency.compute( - record[2], record[4], record[1], - ), - 'balance_all': Currency.compute( - record[2], record[5], record[1], - ), - 'balance_ref': Currency.compute( - record[2], record[5], record[3], - ), - } + result['balance'][record[0]] += Currency.compute( + record[2], record[4], record[1]) + result['balance_all'][record[0]] += Currency.compute( + record[2], record[5], record[1]) + result['balance_ref'][record[0]] += Currency.compute( + record[2], record[5], record[3]) - for name in names: - result[name][record[0]] += values[name] + MemCache.store_result(cashbooks, cache_keys, result) return result @fields.depends('btype') diff --git a/currency.py b/currency.py new file mode 100644 index 0000000..68b1636 --- /dev/null +++ b/currency.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds.de for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + +from trytond.pool import Pool, PoolMeta +from .model import CACHEKEY_CURRENCY + + +class CurrencyRate(metaclass=PoolMeta): + __name__ = 'currency.currency.rate' + + @classmethod + def create(cls, vlist): + """ update cache-value + """ + MemCache = Pool().get('cashbook.memcache') + + records = super(CurrencyRate, cls).create(vlist) + for rate in records: + MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate) + return records + + @classmethod + def write(cls, *args): + """ update cache-value + """ + MemCache = Pool().get('cashbook.memcache') + + super(CurrencyRate, cls).write(*args) + + actions = iter(args) + for rates, values in zip(actions, actions): + for rate in rates: + MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate) + + @classmethod + def delete(cls, records): + """ set cache to None + """ + MemCache = Pool().get('cashbook.memcache') + + for record in records: + MemCache.record_update(CACHEKEY_CURRENCY % record.currency.id, None) + super(CurrencyRate, cls).delete(records) + +# end diff --git a/line.py b/line.py index 97de80b..e3d1784 100644 --- a/line.py +++ b/line.py @@ -15,7 +15,7 @@ from sql import Literal from sql.functions import DatePart from sql.conditionals import Case from .book import sel_state_book -from .mixin import SecondCurrencyMixin +from .mixin import SecondCurrencyMixin, MemCacheIndexMx sel_payee = [ @@ -48,7 +48,7 @@ STATES = { DEPENDS=['state', 'state_cashbook'] -class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView): +class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView): 'Cashbook Line' __name__ = 'cashbook.line' @@ -64,7 +64,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView): states=STATES, depends=DEPENDS) descr_short = fields.Function(fields.Char(string='Description', readonly=True), 'on_change_with_descr_short', searcher='search_descr_short') - category = fields.Many2One(string='Category', + category = fields.Many2One(string='Category', select=True, model_name='cashbook.category', ondelete='RESTRICT', states={ 'readonly': Or( @@ -88,7 +88,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView): states={'invisible': True}), 'on_change_with_booktransf_feature') bookingtype = fields.Selection(string='Type', required=True, - help='Type of Booking', selection=sel_bookingtype, + help='Type of Booking', selection=sel_bookingtype, select=True, states=STATES, depends=DEPENDS) bookingtype_string = bookingtype.translated('bookingtype') amount = fields.Numeric(string='Amount', digits=(16, Eval('currency_digits', 2)), diff --git a/mixin.py b/mixin.py index 003deb4..33e8fca 100644 --- a/mixin.py +++ b/mixin.py @@ -10,6 +10,7 @@ from trytond.modules.currency.ir import rate_decimal from trytond.transaction import Transaction from decimal import Decimal + STATES = { 'readonly': Or( Eval('state', '') != 'edit', @@ -187,3 +188,18 @@ class SecondCurrencyMixin: return 2 # end SecondCurrencyMixin + + +class MemCacheIndexMx: + """ add index to 'create_date' + 'write_date' + """ + @classmethod + def __setup__(cls): + super(MemCacheIndexMx, cls).__setup__() + # add index + cls.write_date.select = True + cls.create_date.select = True + +# end MemCacheIndexMx + + diff --git a/model.py b/model.py index 342892a..b7010a8 100644 --- a/model.py +++ b/model.py @@ -3,11 +3,18 @@ # The COPYRIGHT file at the top level of this repository contains the # full copyright notices and license terms. -from trytond.model import MultiValueMixin, ValueMixin, fields, Unique +from trytond.model import MultiValueMixin, ValueMixin, fields, Unique, Model from trytond.transaction import Transaction from trytond.pool import Pool +from trytond.cache import MemoryCache +from datetime import timedelta, datetime +from decimal import Decimal from sql import With, Literal from sql.functions import Function +from sql.conditionals import Coalesce + +ENABLE_CACHE = True +CACHEKEY_CURRENCY = 'currency-%s' class ArrayAgg(Function): @@ -62,6 +69,149 @@ class Array(Function): # end Array +class MemCache(Model): + """ store values to cache + """ + __name__ = 'cashbook.memcache' + + _cashbook_value_cache = MemoryCache( + 'cashbook.book.valuecache', + context = False, + duration = timedelta(seconds=60*60*4)) + + @classmethod + def read_value(cls, cache_key): + """ read values from cache + """ + if ENABLE_CACHE == False: + return None + return cls._cashbook_value_cache.get(cache_key) + + @classmethod + def store_result(cls, records, cache_keys, values): + """ store result to cache + """ + if ENABLE_CACHE == False: + return + for record in records: + data = {x:values[x][record.id] + for x in values.keys() + if record.id in values[x].keys()} + cls._cashbook_value_cache.set(cache_keys[record.id], data) + + @classmethod + def store_value(cls, cache_key, values): + """ store values to cache + """ + if ENABLE_CACHE == False: + return + cls._cashbook_value_cache.set(cache_key, values) + + @classmethod + def read_from_cache(cls, records, cache_keys, names, result): + """ get stored values from memcache + """ + if ENABLE_CACHE == False: + return (records, result) + + todo_records = [] + for record in records: + values = cls.read_value(cache_keys[record.id]) + if values: + for name in names: + if name not in values.keys(): + continue + if values[name] is None: + continue + if result[name][record.id] is None: + result[name][record.id] = Decimal('0.0') + result[name][record.id] += values[name] + else : + todo_records.append(record) + return (todo_records, result) + + @classmethod + def get_key_by_record(cls, name, record, query, addkeys=[]): + """ read records to build a cache-key + """ + pool = Pool() + cursor = Transaction().connection.cursor() + + if ENABLE_CACHE == False: + return '-' + + dt1 = datetime.now() + print('-- get_key_by_record:', name, record.id) + fname = [name, str(record.id)] + fname.extend(addkeys) + + # query the last edited record for each item in 'query' + for line in query: + if len(line.keys()) == 0: + continue + + if 'cachekey' in line.keys(): + key = cls.read_value(line['cachekey']) + if key: + fname.append(key) + continue + + Model = pool.get(line['model']) + tab_model = Model.__table__() + + tab_query = Model.search(line['query'], query=True) + qu1 = tab_model.join(tab_query, + condition=tab_query.id==tab_model.id, + ).select( + tab_model.id, + tab_model.write_date, + tab_model.create_date, + limit = 1, + order_by = [ + Coalesce(tab_model.write_date, tab_model.create_date).desc, + tab_model.id.desc, + ], + ) + cursor.execute(*qu1) + records = cursor.fetchall() + if len(records) > 0: + fname.append(cls.genkey( + records[0][0], + records[0][1], + records[0][2], + )) + else : + fname.append('0') + + if 'cachekey' in line.keys(): + key = cls.store_value(line['cachekey'], fname[-1]) + + print('-- get_key_by_record-END:', datetime.now() - dt1, '-'.join(fname)) + return '-'.join(fname) + + @classmethod + def genkey(cls, id_record, write_date, create_date): + """ get key as text + """ + date_val = write_date if write_date is not None else create_date + return '-'.join([ + str(id_record), + '%s%s' % ( + 'w' if write_date is not None else 'c', + date_val.timestamp() if date_val is not None else '-'), + ]) + + @classmethod + def record_update(cls, cache_key, record): + """ update cache-value + """ + cls.store_value(cache_key, + cls.genkey(record.id, record.write_date, record.create_date) + if record is not None else None) + +# end mem_cache + + def sub_ids_hierarchical(model_name): """ get table with id and sub-ids """ diff --git a/reconciliation.py b/reconciliation.py index 37da8a8..8978e6c 100644 --- a/reconciliation.py +++ b/reconciliation.py @@ -52,7 +52,7 @@ class Reconciliation(Workflow, ModelSQL, ModelView): ], states=STATES, depends=DEPENDS+['date_to']) date_to = fields.Date(string='End Date', - required=True, + required=True, select=True, domain=[ If(Eval('date_to') & Eval('date_from'), ('date_from', '<=', Eval('date_to')), diff --git a/splitline.py b/splitline.py index 3f73531..0c7ad63 100644 --- a/splitline.py +++ b/splitline.py @@ -12,7 +12,7 @@ from trytond.i18n import gettext from trytond.transaction import Transaction from .line import sel_linetype, sel_bookingtype, STATES, DEPENDS from .book import sel_state_book -from .mixin import SecondCurrencyMixin +from .mixin import SecondCurrencyMixin, MemCacheIndexMx sel_linetype = [ @@ -26,7 +26,7 @@ sel_target = [ ] -class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView): +class SplitLine(SecondCurrencyMixin, MemCacheIndexMx, ModelSQL, ModelView): 'Split booking line' __name__ = 'cashbook.split' @@ -37,8 +37,8 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView): states=STATES, depends=DEPENDS) splittype = fields.Selection(string='Type', required=True, help='Type of split booking line', selection=sel_linetype, - states=STATES, depends=DEPENDS) - category = fields.Many2One(string='Category', + states=STATES, depends=DEPENDS, select=True) + category = fields.Many2One(string='Category', select=True, model_name='cashbook.category', ondelete='RESTRICT', states={ 'readonly': STATES['readonly'], @@ -59,7 +59,7 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView): ('owner.id', '=', Eval('owner_cashbook', -1)), ('id', '!=', Eval('cashbook', -1)), ('btype', '!=', None), - ], + ], select=True, states={ 'readonly': STATES['readonly'], 'invisible': Eval('splittype', '') != 'tr', diff --git a/tests/__init__.py b/tests/__init__.py index dea659d..42d67f9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,12 +12,14 @@ from trytond.modules.cashbook.tests.test_config import ConfigTestCase from trytond.modules.cashbook.tests.test_category import CategoryTestCase from trytond.modules.cashbook.tests.test_reconciliation import ReconTestCase from trytond.modules.cashbook.tests.test_bookingwiz import BookingWizardTestCase +from trytond.modules.cashbook.tests.test_currency import CurrencyTestCase __all__ = ['suite'] class CashbookTestCase(\ + CurrencyTestCase, \ BookingWizardTestCase,\ ReconTestCase,\ CategoryTestCase,\ diff --git a/tests/test_currency.py b/tests/test_currency.py new file mode 100644 index 0000000..73f2119 --- /dev/null +++ b/tests/test_currency.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + +from trytond.tests.test_tryton import ModuleTestCase, with_transaction +from trytond.pool import Pool +from trytond.transaction import Transaction +from trytond.modules.cashbook.model import CACHEKEY_CURRENCY +from datetime import date +from decimal import Decimal +import time + + +class CurrencyTestCase(ModuleTestCase): + 'Test cache for currency' + module = 'cashbook' + + @with_transaction() + def test_currency_update_cache(self): + """ add/update/del rate of currency, check cache + """ + pool = Pool() + MemCache = pool.get('cashbook.memcache') + Currency = pool.get('currency.currency') + CurrencyRate = pool.get('currency.currency.rate') + + self.prep_config() + self.prep_company() + + MemCache._cashbook_value_cache.clear_all() + currency, = Currency.search([('name', '=', 'usd')]) + + cache_key = CACHEKEY_CURRENCY % currency.id + + # cache should be empty + self.assertEqual(MemCache.read_value(cache_key), None) + CurrencyRate.delete(currency.rates) + self.assertEqual(len(currency.rates), 0) + + # add rate + Currency.write(*[ + [currency], + { + 'rates': [('create', [{ + 'date': date(2022, 5, 1), + 'rate': Decimal('1.05'), + }])], + }]) + self.assertEqual(len(currency.rates), 1) + + # expected key + value = '%d-c%s' % ( + currency.rates[0].id, + str(currency.rates[0].create_date.timestamp())) + self.assertEqual(MemCache.read_value(cache_key), value) + time.sleep(1.0) + + Currency.write(*[ + [currency], + { + 'rates': [('write', + [currency.rates[0].id], + { + 'rate': Decimal('1.06'), + })], + }]) + self.assertEqual(len(currency.rates), 1) + + value = '%d-w%s' % ( + currency.rates[0].id, + str(currency.rates[0].write_date.timestamp())) + self.assertEqual(MemCache.read_value(cache_key), value) + + Currency.write(*[ + [currency], + { + 'rates': [('delete', [currency.rates[0].id])], + }]) + self.assertEqual(MemCache.read_value(cache_key), None) + +# end CurrencyTestCase diff --git a/types.py b/types.py index a2554c4..c86cc02 100644 --- a/types.py +++ b/types.py @@ -17,7 +17,7 @@ class Type(ModelSQL, ModelView): company = fields.Many2One(string='Company', model_name='company.company', required=True, ondelete="RESTRICT") feature = fields.Selection(string='Feature', required=True, - selection='get_sel_feature', + selection='get_sel_feature', select=True, help='Select feature set of the Cashbook.') @classmethod