speedup: indexes, caching

__init__.py

@@ -15,11 +15,15 @@ from .configuration import Configuration, UserConfiguration
 from .category import Category
 from .reconciliation import Reconciliation
 from .cbreport import ReconciliationReport
+from .currency import CurrencyRate
+from .model import MemCache
 
 def register():
     Pool.register(
+        MemCache,
         Configuration,
         UserConfiguration,
+        CurrencyRate,
         Type,
         Category,
         Book,

book.py

@@ -13,9 +13,9 @@ from trytond.report import Report
 from decimal import Decimal
 from datetime import date
 from sql.aggregate import Sum
-from sql.conditionals import Case, Coalesce
+from sql.conditionals import Case
-from sql.functions import CurrentDate
+from .model import order_name_hierarchical, sub_ids_hierarchical, \
-from .model import order_name_hierarchical, sub_ids_hierarchical, AnyInArray
+    AnyInArray, CACHEKEY_CURRENCY
 
 
 STATES = {
@@ -48,12 +48,12 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
     __name__ = 'cashbook.book'
 
     company = fields.Many2One(string='Company', model_name='company.company',
-        required=True, ondelete="RESTRICT")
+        required=True, select=True, ondelete="RESTRICT")
     name = fields.Char(string='Name', required=True,
         states=STATES, depends=DEPENDS)
     description = fields.Text(string='Description',
         states=STATES, depends=DEPENDS)
-    btype = fields.Many2One(string='Type',
+    btype = fields.Many2One(string='Type', select=True,
         help='A cash book with type can contain postings. Without type is a view.',
         model_name='cashbook.type', ondelete='RESTRICT',
         states={
@@ -346,12 +346,15 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
         Currency = pool.get('currency.currency')
         Company = pool.get('company.company')
         IrDate = pool.get('ir.date')
+        MemCache = pool.get('cashbook.memcache')
         tab_book = Book2.__table__()
         tab_comp = Company.__table__()
         cursor = Transaction().connection.cursor()
         context = Transaction().context
 
-        result = {x:{y.id: Decimal('0.0') for y in cashbooks} for x in names}
+        result = {
+            x:{y.id: Decimal('0.0') for y in cashbooks}
+                for x in ['balance', 'balance_all', 'balance_ref']}
 
         # deny invalid date in context
         query_date = context.get('date', IrDate.today())
@@ -361,6 +364,28 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
             except :
                 query_date = IrDate.today()
 
+        cache_keys = {
+                x.id: MemCache.get_key_by_record(
+                    name = 'get_balance_cashbook',
+                    record = x,
+                    query = [{
+                            'model': 'cashbook.line',
+                            'query': [('cashbook.parent', 'child_of', [x.id])],
+                        }, {
+                            'model': 'currency.currency.rate',
+                            'query': [('currency.id', '=', x.currency.id)],
+                            'cachekey': CACHEKEY_CURRENCY % x.currency.id,
+                        }, ],
+                    addkeys = [query_date.isoformat()])
+                for x in cashbooks
+            }
+
+        # read from cache
+        (todo_cashbook, result) = MemCache.read_from_cache(
+            cashbooks, cache_keys, names, result)
+        if len(todo_cashbook) == 0:
+            return result
+
         # query balances of cashbooks and sub-cashbooks
         with Transaction().set_context({
             'date': query_date,
@@ -381,26 +406,20 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
                     Sum(tab_line.balance).as_('balance'),
                     Sum(tab_line.balance_all).as_('balance_all'),
                     group_by=[tab_book.id, tab_line.currency, tab_comp.currency],
-                    where=tab_book.id.in_([x.id for x in cashbooks]),
+                    where=tab_book.id.in_([x.id for x in todo_cashbook]),
                 )
             cursor.execute(*query)
             records = cursor.fetchall()
 
             for record in records:
-                values = {
+                result['balance'][record[0]] += Currency.compute(
-                    'balance': Currency.compute(
+                        record[2], record[4], record[1])
-                            record[2], record[4], record[1],
+                result['balance_all'][record[0]] += Currency.compute(
-                        ),
+                        record[2], record[5], record[1])
-                    'balance_all': Currency.compute(
+                result['balance_ref'][record[0]] += Currency.compute(
-                            record[2], record[5], record[1],
+                        record[2], record[5], record[3])
-                        ),
-                    'balance_ref': Currency.compute(
-                            record[2], record[5], record[3],
-                        ),
-                    }
 
-                for name in names:
+        MemCache.store_result(cashbooks, cache_keys, result)
-                    result[name][record[0]] += values[name]
         return result
 
     @fields.depends('btype')

currency.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# This file is part of the cashbook-module from m-ds.de for Tryton.
+# The COPYRIGHT file at the top level of this repository contains the
+# full copyright notices and license terms.
+
+from trytond.pool import Pool, PoolMeta
+from .model import CACHEKEY_CURRENCY
+
+
+class CurrencyRate(metaclass=PoolMeta):
+    __name__ = 'currency.currency.rate'
+
+    @classmethod
+    def create(cls, vlist):
+        """ update cache-value
+        """
+        MemCache = Pool().get('cashbook.memcache')
+
+        records = super(CurrencyRate, cls).create(vlist)
+        for rate in records:
+            MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate)
+        return records
+
+    @classmethod
+    def write(cls, *args):
+        """ update cache-value
+        """
+        MemCache = Pool().get('cashbook.memcache')
+
+        super(CurrencyRate, cls).write(*args)
+
+        actions = iter(args)
+        for rates, values in zip(actions, actions):
+            for rate in rates:
+                MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate)
+
+    @classmethod
+    def delete(cls, records):
+        """ set cache to None
+        """
+        MemCache = Pool().get('cashbook.memcache')
+
+        for record in records:
+            MemCache.record_update(CACHEKEY_CURRENCY % record.currency.id, None)
+        super(CurrencyRate, cls).delete(records)
+
+# end

line.py

@@ -15,7 +15,7 @@ from sql import Literal
 from sql.functions import DatePart
 from sql.conditionals import Case
 from .book import sel_state_book
-from .mixin import SecondCurrencyMixin
+from .mixin import SecondCurrencyMixin, MemCacheIndexMx
 
 
 sel_payee = [
@@ -48,7 +48,7 @@ STATES = {
 DEPENDS=['state', 'state_cashbook']
 
 
-class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
+class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView):
     'Cashbook Line'
     __name__ = 'cashbook.line'
 
@@ -64,7 +64,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
         states=STATES, depends=DEPENDS)
     descr_short = fields.Function(fields.Char(string='Description', readonly=True),
         'on_change_with_descr_short', searcher='search_descr_short')
-    category = fields.Many2One(string='Category',
+    category = fields.Many2One(string='Category', select=True,
         model_name='cashbook.category', ondelete='RESTRICT',
         states={
             'readonly': Or(
@@ -88,7 +88,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
         states={'invisible': True}), 'on_change_with_booktransf_feature')
 
     bookingtype = fields.Selection(string='Type', required=True,
-        help='Type of Booking', selection=sel_bookingtype,
+        help='Type of Booking', selection=sel_bookingtype, select=True,
         states=STATES, depends=DEPENDS)
     bookingtype_string = bookingtype.translated('bookingtype')
     amount = fields.Numeric(string='Amount', digits=(16, Eval('currency_digits', 2)),

mixin.py

@@ -10,6 +10,7 @@ from trytond.modules.currency.ir import rate_decimal
 from trytond.transaction import Transaction
 from decimal import Decimal
 
+
 STATES = {
     'readonly': Or(
             Eval('state', '') != 'edit',
@@ -187,3 +188,18 @@ class SecondCurrencyMixin:
             return 2
 
 # end SecondCurrencyMixin
+
+
+class MemCacheIndexMx:
+    """ add index to 'create_date' + 'write_date'
+    """
+    @classmethod
+    def __setup__(cls):
+        super(MemCacheIndexMx, cls).__setup__()
+        # add index
+        cls.write_date.select = True
+        cls.create_date.select = True
+
+# end MemCacheIndexMx
+
+

model.py

@@ -3,11 +3,18 @@
 # The COPYRIGHT file at the top level of this repository contains the
 # full copyright notices and license terms.
 
-from trytond.model import MultiValueMixin, ValueMixin, fields, Unique
+from trytond.model import MultiValueMixin, ValueMixin, fields, Unique, Model
 from trytond.transaction import Transaction
 from trytond.pool import Pool
+from trytond.cache import MemoryCache
+from datetime import timedelta, datetime
+from decimal import Decimal
 from sql import With, Literal
 from sql.functions import Function
+from sql.conditionals import Coalesce
+
+ENABLE_CACHE = True
+CACHEKEY_CURRENCY = 'currency-%s'
 
 
 class ArrayAgg(Function):
@@ -62,6 +69,149 @@ class Array(Function):
 # end Array
 
 
+class MemCache(Model):
+    """ store values to cache
+    """
+    __name__ = 'cashbook.memcache'
+
+    _cashbook_value_cache = MemoryCache(
+        'cashbook.book.valuecache',
+        context = False,
+        duration = timedelta(seconds=60*60*4))
+
+    @classmethod
+    def read_value(cls, cache_key):
+        """ read values from cache
+        """
+        if ENABLE_CACHE == False:
+            return None
+        return cls._cashbook_value_cache.get(cache_key)
+
+    @classmethod
+    def store_result(cls, records, cache_keys, values):
+        """ store result to cache
+        """
+        if ENABLE_CACHE == False:
+            return
+        for record in records:
+            data = {x:values[x][record.id]
+                        for x in values.keys()
+                            if record.id in values[x].keys()}
+            cls._cashbook_value_cache.set(cache_keys[record.id], data)
+
+    @classmethod
+    def store_value(cls, cache_key, values):
+        """ store values to cache
+        """
+        if ENABLE_CACHE == False:
+            return
+        cls._cashbook_value_cache.set(cache_key, values)
+
+    @classmethod
+    def read_from_cache(cls, records, cache_keys, names, result):
+        """ get stored values from memcache
+        """
+        if ENABLE_CACHE == False:
+            return (records, result)
+
+        todo_records = []
+        for record in records:
+            values = cls.read_value(cache_keys[record.id])
+            if values:
+                for name in names:
+                    if name not in values.keys():
+                        continue
+                    if values[name] is None:
+                        continue
+                    if result[name][record.id] is None:
+                        result[name][record.id] = Decimal('0.0')
+                    result[name][record.id] += values[name]
+            else :
+                todo_records.append(record)
+        return (todo_records, result)
+
+    @classmethod
+    def get_key_by_record(cls, name, record, query, addkeys=[]):
+        """ read records to build a cache-key
+        """
+        pool = Pool()
+        cursor = Transaction().connection.cursor()
+
+        if ENABLE_CACHE == False:
+            return '-'
+
+        dt1 = datetime.now()
+        print('-- get_key_by_record:', name, record.id)
+        fname = [name, str(record.id)]
+        fname.extend(addkeys)
+
+        # query the last edited record for each item in 'query'
+        for line in query:
+            if len(line.keys()) == 0:
+                continue
+
+            if 'cachekey' in line.keys():
+                key = cls.read_value(line['cachekey'])
+                if key:
+                    fname.append(key)
+                    continue
+
+            Model = pool.get(line['model'])
+            tab_model = Model.__table__()
+
+            tab_query = Model.search(line['query'], query=True)
+            qu1 = tab_model.join(tab_query,
+                    condition=tab_query.id==tab_model.id,
+                ).select(
+                    tab_model.id,
+                    tab_model.write_date,
+                    tab_model.create_date,
+                    limit = 1,
+                    order_by = [
+                        Coalesce(tab_model.write_date, tab_model.create_date).desc,
+                        tab_model.id.desc,
+                        ],
+                )
+            cursor.execute(*qu1)
+            records = cursor.fetchall()
+            if len(records) > 0:
+                fname.append(cls.genkey(
+                    records[0][0],
+                    records[0][1],
+                    records[0][2],
+                    ))
+            else :
+                fname.append('0')
+
+            if 'cachekey' in line.keys():
+                key = cls.store_value(line['cachekey'], fname[-1])
+
+        print('-- get_key_by_record-END:', datetime.now() - dt1, '-'.join(fname))
+        return '-'.join(fname)
+
+    @classmethod
+    def genkey(cls, id_record, write_date, create_date):
+        """ get key as text
+        """
+        date_val = write_date if write_date is not None else create_date
+        return '-'.join([
+            str(id_record),
+            '%s%s' % (
+                'w' if write_date is not None else 'c',
+                date_val.timestamp() if date_val is not None else '-'),
+            ])
+
+    @classmethod
+    def record_update(cls, cache_key, record):
+        """ update cache-value
+        """
+        cls.store_value(cache_key,
+            cls.genkey(record.id, record.write_date, record.create_date)
+                if record is not None else None)
+
+# end mem_cache
+
+
 def sub_ids_hierarchical(model_name):
     """ get table with id and sub-ids
     """

reconciliation.py

@@ -52,7 +52,7 @@ class Reconciliation(Workflow, ModelSQL, ModelView):
             ],
         states=STATES, depends=DEPENDS+['date_to'])
     date_to = fields.Date(string='End Date',
-        required=True,
+        required=True, select=True,
         domain=[
             If(Eval('date_to') & Eval('date_from'),
                 ('date_from', '<=', Eval('date_to')),

splitline.py

@@ -12,7 +12,7 @@ from trytond.i18n import gettext
 from trytond.transaction import Transaction
 from .line import sel_linetype, sel_bookingtype, STATES, DEPENDS
 from .book import sel_state_book
-from .mixin import SecondCurrencyMixin
+from .mixin import SecondCurrencyMixin, MemCacheIndexMx
 
 
 sel_linetype = [
@@ -26,7 +26,7 @@ sel_target = [
     ]
 
 
-class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
+class SplitLine(SecondCurrencyMixin, MemCacheIndexMx, ModelSQL, ModelView):
     'Split booking line'
     __name__ = 'cashbook.split'
 
@@ -37,8 +37,8 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
         states=STATES, depends=DEPENDS)
     splittype = fields.Selection(string='Type', required=True,
         help='Type of split booking line', selection=sel_linetype,
-        states=STATES, depends=DEPENDS)
+        states=STATES, depends=DEPENDS, select=True)
-    category = fields.Many2One(string='Category',
+    category = fields.Many2One(string='Category', select=True,
         model_name='cashbook.category', ondelete='RESTRICT',
         states={
             'readonly': STATES['readonly'],
@@ -59,7 +59,7 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
             ('owner.id', '=', Eval('owner_cashbook', -1)),
             ('id', '!=', Eval('cashbook', -1)),
             ('btype', '!=', None),
-            ],
+            ], select=True,
         states={
             'readonly': STATES['readonly'],
             'invisible': Eval('splittype', '') != 'tr',

tests/__init__.py

@@ -12,12 +12,14 @@ from trytond.modules.cashbook.tests.test_config import ConfigTestCase
 from trytond.modules.cashbook.tests.test_category import CategoryTestCase
 from trytond.modules.cashbook.tests.test_reconciliation import ReconTestCase
 from trytond.modules.cashbook.tests.test_bookingwiz import BookingWizardTestCase
+from trytond.modules.cashbook.tests.test_currency import CurrencyTestCase
 
 
 __all__ = ['suite']
 
 
 class CashbookTestCase(\
+    CurrencyTestCase, \
     BookingWizardTestCase,\
     ReconTestCase,\
     CategoryTestCase,\

tests/test_currency.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+# This file is part of the cashbook-module from m-ds for Tryton.
+# The COPYRIGHT file at the top level of this repository contains the
+# full copyright notices and license terms.
+
+from trytond.tests.test_tryton import ModuleTestCase, with_transaction
+from trytond.pool import Pool
+from trytond.transaction import Transaction
+from trytond.modules.cashbook.model import CACHEKEY_CURRENCY
+from datetime import date
+from decimal import Decimal
+import time
+
+
+class CurrencyTestCase(ModuleTestCase):
+    'Test cache for currency'
+    module = 'cashbook'
+
+    @with_transaction()
+    def test_currency_update_cache(self):
+        """ add/update/del rate of currency, check cache
+        """
+        pool = Pool()
+        MemCache = pool.get('cashbook.memcache')
+        Currency = pool.get('currency.currency')
+        CurrencyRate = pool.get('currency.currency.rate')
+
+        self.prep_config()
+        self.prep_company()
+
+        MemCache._cashbook_value_cache.clear_all()
+        currency, = Currency.search([('name', '=', 'usd')])
+
+        cache_key = CACHEKEY_CURRENCY % currency.id
+
+        # cache should be empty
+        self.assertEqual(MemCache.read_value(cache_key), None)
+        CurrencyRate.delete(currency.rates)
+        self.assertEqual(len(currency.rates), 0)
+
+        # add rate
+        Currency.write(*[
+            [currency],
+            {
+                'rates': [('create', [{
+                    'date': date(2022, 5, 1),
+                    'rate': Decimal('1.05'),
+                    }])],
+            }])
+        self.assertEqual(len(currency.rates), 1)
+
+        # expected key
+        value = '%d-c%s' % (
+            currency.rates[0].id,
+            str(currency.rates[0].create_date.timestamp()))
+        self.assertEqual(MemCache.read_value(cache_key), value)
+        time.sleep(1.0)
+
+        Currency.write(*[
+            [currency],
+            {
+                'rates': [('write',
+                    [currency.rates[0].id],
+                    {
+                        'rate': Decimal('1.06'),
+                    })],
+            }])
+        self.assertEqual(len(currency.rates), 1)
+
+        value = '%d-w%s' % (
+            currency.rates[0].id,
+            str(currency.rates[0].write_date.timestamp()))
+        self.assertEqual(MemCache.read_value(cache_key), value)
+
+        Currency.write(*[
+            [currency],
+            {
+                'rates': [('delete', [currency.rates[0].id])],
+            }])
+        self.assertEqual(MemCache.read_value(cache_key), None)
+
+# end CurrencyTestCase

types.py

@@ -17,7 +17,7 @@ class Type(ModelSQL, ModelView):
     company = fields.Many2One(string='Company', model_name='company.company',
         required=True, ondelete="RESTRICT")
     feature = fields.Selection(string='Feature', required=True,
-        selection='get_sel_feature',
+        selection='get_sel_feature', select=True,
         help='Select feature set of the Cashbook.')
 
     @classmethod