mdsfred/cashbook
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

speedup: indexes, caching

Browse source
This commit is contained in:
Frederik Jaeckel 2023-02-26 22:49:21 +01:00
parent 86e6c33cc1
commit 624a5bff55
11 changed files with 352 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
__init__.py
View file

@ -15,11 +15,15 @@ from .configuration import Configuration, UserConfiguration
from .category import Category
from .reconciliation import Reconciliation
from .cbreport import ReconciliationReport
from .currency import CurrencyRate
from .model import MemCache
def register():
Pool.register(
MemCache,
Configuration,
UserConfiguration,
CurrencyRate,
Type,
Category,
Book,

59
book.py
View file

@ -13,9 +13,9 @@ from trytond.report import Report
from decimal import Decimal
from datetime import date
from sql.aggregate import Sum
from sql.conditionals import Case, Coalesce
from sql.functions import CurrentDate
from .model import order_name_hierarchical, sub_ids_hierarchical, AnyInArray
from sql.conditionals import Case
from .model import order_name_hierarchical, sub_ids_hierarchical, \
AnyInArray, CACHEKEY_CURRENCY
STATES = {
@ -48,12 +48,12 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
__name__ = 'cashbook.book'
company = fields.Many2One(string='Company', model_name='company.company',
required=True, ondelete="RESTRICT")
required=True, select=True, ondelete="RESTRICT")
name = fields.Char(string='Name', required=True,
states=STATES, depends=DEPENDS)
description = fields.Text(string='Description',
states=STATES, depends=DEPENDS)
btype = fields.Many2One(string='Type',
btype = fields.Many2One(string='Type', select=True,
help='A cash book with type can contain postings. Without type is a view.',
model_name='cashbook.type', ondelete='RESTRICT',
states={
@ -346,12 +346,15 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
Currency = pool.get('currency.currency')
Company = pool.get('company.company')
IrDate = pool.get('ir.date')
MemCache = pool.get('cashbook.memcache')
tab_book = Book2.__table__()
tab_comp = Company.__table__()
cursor = Transaction().connection.cursor()
context = Transaction().context
result = {x:{y.id: Decimal('0.0') for y in cashbooks} for x in names}
result = {
x:{y.id: Decimal('0.0') for y in cashbooks}
for x in ['balance', 'balance_all', 'balance_ref']}
# deny invalid date in context
query_date = context.get('date', IrDate.today())
@ -361,6 +364,28 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
except :
query_date = IrDate.today()
cache_keys = {
x.id: MemCache.get_key_by_record(
name = 'get_balance_cashbook',
record = x,
query = [{
'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])],
}, {
'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)],
'cachekey': CACHEKEY_CURRENCY % x.currency.id,
}, ],
addkeys = [query_date.isoformat()])
for x in cashbooks
}
# read from cache
(todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0:
return result
# query balances of cashbooks and sub-cashbooks
with Transaction().set_context({
'date': query_date,
@ -381,26 +406,20 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
Sum(tab_line.balance).as_('balance'),
Sum(tab_line.balance_all).as_('balance_all'),
group_by=[tab_book.id, tab_line.currency, tab_comp.currency],
where=tab_book.id.in_([x.id for x in cashbooks]),
where=tab_book.id.in_([x.id for x in todo_cashbook]),
)
cursor.execute(*query)
records = cursor.fetchall()
for record in records:
values = {
'balance': Currency.compute(
record[2], record[4], record[1],
),
'balance_all': Currency.compute(
record[2], record[5], record[1],
),
'balance_ref': Currency.compute(
record[2], record[5], record[3],
),
}
result['balance'][record[0]] += Currency.compute(
record[2], record[4], record[1])
result['balance_all'][record[0]] += Currency.compute(
record[2], record[5], record[1])
result['balance_ref'][record[0]] += Currency.compute(
record[2], record[5], record[3])
for name in names:
result[name][record[0]] += values[name]
MemCache.store_result(cashbooks, cache_keys, result)
return result
@fields.depends('btype')

47
currency.py Normal file
View file

@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
# This file is part of the cashbook-module from m-ds.de for Tryton.
# The COPYRIGHT file at the top level of this repository contains the
# full copyright notices and license terms.
from trytond.pool import Pool, PoolMeta
from .model import CACHEKEY_CURRENCY
class CurrencyRate(metaclass=PoolMeta):
__name__ = 'currency.currency.rate'
@classmethod
def create(cls, vlist):
""" update cache-value
"""
MemCache = Pool().get('cashbook.memcache')
records = super(CurrencyRate, cls).create(vlist)
for rate in records:
MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate)
return records
@classmethod
def write(cls, *args):
""" update cache-value
"""
MemCache = Pool().get('cashbook.memcache')
super(CurrencyRate, cls).write(*args)
actions = iter(args)
for rates, values in zip(actions, actions):
for rate in rates:
MemCache.record_update(CACHEKEY_CURRENCY % rate.currency.id, rate)
@classmethod
def delete(cls, records):
""" set cache to None
"""
MemCache = Pool().get('cashbook.memcache')
for record in records:
MemCache.record_update(CACHEKEY_CURRENCY % record.currency.id, None)
super(CurrencyRate, cls).delete(records)
# end

8
line.py
View file

@ -15,7 +15,7 @@ from sql import Literal
from sql.functions import DatePart
from sql.conditionals import Case
from .book import sel_state_book
from .mixin import SecondCurrencyMixin
from .mixin import SecondCurrencyMixin, MemCacheIndexMx
sel_payee = [
@ -48,7 +48,7 @@ STATES = {
DEPENDS=['state', 'state_cashbook']
class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView):
'Cashbook Line'
__name__ = 'cashbook.line'
@ -64,7 +64,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
states=STATES, depends=DEPENDS)
descr_short = fields.Function(fields.Char(string='Description', readonly=True),
'on_change_with_descr_short', searcher='search_descr_short')
category = fields.Many2One(string='Category',
category = fields.Many2One(string='Category', select=True,
model_name='cashbook.category', ondelete='RESTRICT',
states={
'readonly': Or(
@ -88,7 +88,7 @@ class Line(SecondCurrencyMixin, Workflow, ModelSQL, ModelView):
states={'invisible': True}), 'on_change_with_booktransf_feature')
bookingtype = fields.Selection(string='Type', required=True,
help='Type of Booking', selection=sel_bookingtype,
help='Type of Booking', selection=sel_bookingtype, select=True,
states=STATES, depends=DEPENDS)
bookingtype_string = bookingtype.translated('bookingtype')
amount = fields.Numeric(string='Amount', digits=(16, Eval('currency_digits', 2)),

16
mixin.py
View file

@ -10,6 +10,7 @@ from trytond.modules.currency.ir import rate_decimal
from trytond.transaction import Transaction
from decimal import Decimal
STATES = {
'readonly': Or(
Eval('state', '') != 'edit',
@ -187,3 +188,18 @@ class SecondCurrencyMixin:
return 2
# end SecondCurrencyMixin
class MemCacheIndexMx:
""" add index to 'create_date' + 'write_date'
"""
@classmethod
def __setup__(cls):
super(MemCacheIndexMx, cls).__setup__()
# add index
cls.write_date.select = True
cls.create_date.select = True
# end MemCacheIndexMx

152
model.py
View file

@ -3,11 +3,18 @@
# The COPYRIGHT file at the top level of this repository contains the
# full copyright notices and license terms.
from trytond.model import MultiValueMixin, ValueMixin, fields, Unique
from trytond.model import MultiValueMixin, ValueMixin, fields, Unique, Model
from trytond.transaction import Transaction
from trytond.pool import Pool
from trytond.cache import MemoryCache
from datetime import timedelta, datetime
from decimal import Decimal
from sql import With, Literal
from sql.functions import Function
from sql.conditionals import Coalesce
ENABLE_CACHE = True
CACHEKEY_CURRENCY = 'currency-%s'
class ArrayAgg(Function):
@ -62,6 +69,149 @@ class Array(Function):
# end Array
class MemCache(Model):
""" store values to cache
"""
__name__ = 'cashbook.memcache'
_cashbook_value_cache = MemoryCache(
'cashbook.book.valuecache',
context = False,
duration = timedelta(seconds=60*60*4))
@classmethod
def read_value(cls, cache_key):
""" read values from cache
"""
if ENABLE_CACHE == False:
return None
return cls._cashbook_value_cache.get(cache_key)
@classmethod
def store_result(cls, records, cache_keys, values):
""" store result to cache
"""
if ENABLE_CACHE == False:
return
for record in records:
data = {x:values[x][record.id]
for x in values.keys()
if record.id in values[x].keys()}
cls._cashbook_value_cache.set(cache_keys[record.id], data)
@classmethod
def store_value(cls, cache_key, values):
""" store values to cache
"""
if ENABLE_CACHE == False:
return
cls._cashbook_value_cache.set(cache_key, values)
@classmethod
def read_from_cache(cls, records, cache_keys, names, result):
""" get stored values from memcache
"""
if ENABLE_CACHE == False:
return (records, result)
todo_records = []
for record in records:
values = cls.read_value(cache_keys[record.id])
if values:
for name in names:
if name not in values.keys():
continue
if values[name] is None:
continue
if result[name][record.id] is None:
result[name][record.id] = Decimal('0.0')
result[name][record.id] += values[name]
else :
todo_records.append(record)
return (todo_records, result)
@classmethod
def get_key_by_record(cls, name, record, query, addkeys=[]):
""" read records to build a cache-key
"""
pool = Pool()
cursor = Transaction().connection.cursor()
if ENABLE_CACHE == False:
return '-'
dt1 = datetime.now()
print('-- get_key_by_record:', name, record.id)
fname = [name, str(record.id)]
fname.extend(addkeys)
# query the last edited record for each item in 'query'
for line in query:
if len(line.keys()) == 0:
continue
if 'cachekey' in line.keys():
key = cls.read_value(line['cachekey'])
if key:
fname.append(key)
continue
Model = pool.get(line['model'])
tab_model = Model.__table__()
tab_query = Model.search(line['query'], query=True)
qu1 = tab_model.join(tab_query,
condition=tab_query.id==tab_model.id,
).select(
tab_model.id,
tab_model.write_date,
tab_model.create_date,
limit = 1,
order_by = [
Coalesce(tab_model.write_date, tab_model.create_date).desc,
tab_model.id.desc,
],
)
cursor.execute(*qu1)
records = cursor.fetchall()
if len(records) > 0:
fname.append(cls.genkey(
records[0][0],
records[0][1],
records[0][2],
))
else :
fname.append('0')
if 'cachekey' in line.keys():
key = cls.store_value(line['cachekey'], fname[-1])
print('-- get_key_by_record-END:', datetime.now() - dt1, '-'.join(fname))
return '-'.join(fname)
@classmethod
def genkey(cls, id_record, write_date, create_date):
""" get key as text
"""
date_val = write_date if write_date is not None else create_date
return '-'.join([
str(id_record),
'%s%s' % (
'w' if write_date is not None else 'c',
date_val.timestamp() if date_val is not None else '-'),
])
@classmethod
def record_update(cls, cache_key, record):
""" update cache-value
"""
cls.store_value(cache_key,
cls.genkey(record.id, record.write_date, record.create_date)
if record is not None else None)
# end mem_cache
def sub_ids_hierarchical(model_name):
""" get table with id and sub-ids
"""

2
reconciliation.py
View file

@ -52,7 +52,7 @@ class Reconciliation(Workflow, ModelSQL, ModelView):
],
states=STATES, depends=DEPENDS+['date_to'])
date_to = fields.Date(string='End Date',
required=True,
required=True, select=True,
domain=[
If(Eval('date_to') & Eval('date_from'),
('date_from', '<=', Eval('date_to')),

10
splitline.py
View file

@ -12,7 +12,7 @@ from trytond.i18n import gettext
from trytond.transaction import Transaction
from .line import sel_linetype, sel_bookingtype, STATES, DEPENDS
from .book import sel_state_book
from .mixin import SecondCurrencyMixin
from .mixin import SecondCurrencyMixin, MemCacheIndexMx
sel_linetype = [
@ -26,7 +26,7 @@ sel_target = [
]
class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
class SplitLine(SecondCurrencyMixin, MemCacheIndexMx, ModelSQL, ModelView):
'Split booking line'
__name__ = 'cashbook.split'
@ -37,8 +37,8 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
states=STATES, depends=DEPENDS)
splittype = fields.Selection(string='Type', required=True,
help='Type of split booking line', selection=sel_linetype,
states=STATES, depends=DEPENDS)
category = fields.Many2One(string='Category',
states=STATES, depends=DEPENDS, select=True)
category = fields.Many2One(string='Category', select=True,
model_name='cashbook.category', ondelete='RESTRICT',
states={
'readonly': STATES['readonly'],
@ -59,7 +59,7 @@ class SplitLine(SecondCurrencyMixin, ModelSQL, ModelView):
('owner.id', '=', Eval('owner_cashbook', -1)),
('id', '!=', Eval('cashbook', -1)),
('btype', '!=', None),
],
], select=True,
states={
'readonly': STATES['readonly'],
'invisible': Eval('splittype', '') != 'tr',

2
tests/__init__.py
View file

@ -12,12 +12,14 @@ from trytond.modules.cashbook.tests.test_config import ConfigTestCase
from trytond.modules.cashbook.tests.test_category import CategoryTestCase
from trytond.modules.cashbook.tests.test_reconciliation import ReconTestCase
from trytond.modules.cashbook.tests.test_bookingwiz import BookingWizardTestCase
from trytond.modules.cashbook.tests.test_currency import CurrencyTestCase
__all__ = ['suite']
class CashbookTestCase(\
CurrencyTestCase, \
BookingWizardTestCase,\
ReconTestCase,\
CategoryTestCase,\

82
tests/test_currency.py Normal file
View file

@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
# This file is part of the cashbook-module from m-ds for Tryton.
# The COPYRIGHT file at the top level of this repository contains the
# full copyright notices and license terms.
from trytond.tests.test_tryton import ModuleTestCase, with_transaction
from trytond.pool import Pool
from trytond.transaction import Transaction
from trytond.modules.cashbook.model import CACHEKEY_CURRENCY
from datetime import date
from decimal import Decimal
import time
class CurrencyTestCase(ModuleTestCase):
'Test cache for currency'
module = 'cashbook'
@with_transaction()
def test_currency_update_cache(self):
""" add/update/del rate of currency, check cache
"""
pool = Pool()
MemCache = pool.get('cashbook.memcache')
Currency = pool.get('currency.currency')
CurrencyRate = pool.get('currency.currency.rate')
self.prep_config()
self.prep_company()
MemCache._cashbook_value_cache.clear_all()
currency, = Currency.search([('name', '=', 'usd')])
cache_key = CACHEKEY_CURRENCY % currency.id
# cache should be empty
self.assertEqual(MemCache.read_value(cache_key), None)
CurrencyRate.delete(currency.rates)
self.assertEqual(len(currency.rates), 0)
# add rate
Currency.write(*[
[currency],
{
'rates': [('create', [{
'date': date(2022, 5, 1),
'rate': Decimal('1.05'),
}])],
}])
self.assertEqual(len(currency.rates), 1)
# expected key
value = '%d-c%s' % (
currency.rates[0].id,
str(currency.rates[0].create_date.timestamp()))
self.assertEqual(MemCache.read_value(cache_key), value)
time.sleep(1.0)
Currency.write(*[
[currency],
{
'rates': [('write',
[currency.rates[0].id],
{
'rate': Decimal('1.06'),
})],
}])
self.assertEqual(len(currency.rates), 1)
value = '%d-w%s' % (
currency.rates[0].id,
str(currency.rates[0].write_date.timestamp()))
self.assertEqual(MemCache.read_value(cache_key), value)
Currency.write(*[
[currency],
{
'rates': [('delete', [currency.rates[0].id])],
}])
self.assertEqual(MemCache.read_value(cache_key), None)
# end CurrencyTestCase

2
types.py
View file

@ -17,7 +17,7 @@ class Type(ModelSQL, ModelView):
company = fields.Many2One(string='Company', model_name='company.company',
required=True, ondelete="RESTRICT")
feature = fields.Selection(string='Feature', required=True,
selection='get_sel_feature',
selection='get_sel_feature', select=True,
help='Select feature set of the Cashbook.')
@classmethod