mdsfred/cashbook
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

book/line: logging, model: cache-write skip existing values

Browse source
This commit is contained in:
Frederik Jaeckel 2023-03-04 21:24:19 +01:00
parent 440e4c66d5
commit 0a5c0585a2
4 changed files with 81 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
book.py
View file

@ -11,12 +11,15 @@ from trytond.transaction import Transaction
from trytond.pool import Pool from trytond.pool import Pool
from trytond.report import Report from trytond.report import Report
from decimal import Decimal from decimal import Decimal
from datetime import date from datetime import date, datetime
from sql.aggregate import Sum from sql.aggregate import Sum
from sql.conditionals import Case from sql.conditionals import Case
from .model import order_name_hierarchical, sub_ids_hierarchical, \ from .model import order_name_hierarchical, sub_ids_hierarchical, \
AnyInArray, CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK AnyInArray, CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK
import logging
logger = logging.getLogger(__name__)
STATES = { STATES = {
'readonly': Eval('state', '') != 'open', 'readonly': Eval('state', '') != 'open',
@ -262,15 +265,15 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
} }
return recname return recname
def get_cachekeys_by_hierarchy(self): # ~ def get_cachekeys_by_hierarchy(self):
""" generate keys for all cashbooks above us # ~ """ generate keys for all cashbooks above us
""" # ~ """
CBook = Pool().get('cashbook.book') # ~ CBook = Pool().get('cashbook.book')
return [CACHEKEY_CASHBOOK % x.id # ~ return [CACHEKEY_CASHBOOK % x.id
for x in CBook.search([ # ~ for x in CBook.search([
('parent', 'parent_of', [self.id]), # ~ ('parent', 'parent_of', [self.id]),
], order=[('id', 'ASC')])] # ~ ], order=[('id', 'ASC')])]
@classmethod @classmethod
def get_balance_of_cashbook_sql(cls): def get_balance_of_cashbook_sql(cls):
@ -362,6 +365,12 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
cursor = Transaction().connection.cursor() cursor = Transaction().connection.cursor()
context = Transaction().context context = Transaction().context
logger.warning('## get_balance_cashbook-GO %(time)s %(ids)s %(name)s' % {
'time': datetime.now().isoformat(),
'ids': str([x.id for x in cashbooks]),
'name': names,
})
result = { result = {
x:{y.id: Decimal('0.0') for y in cashbooks} x:{y.id: Decimal('0.0') for y in cashbooks}
for x in ['balance', 'balance_all', 'balance_ref']} for x in ['balance', 'balance_all', 'balance_ref']}
@ -381,7 +390,7 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
query = [{ query = [{
'model': 'cashbook.line', 'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])], 'query': [('cashbook.parent', 'child_of', [x.id])],
'cachekey': CACHEKEY_CASHBOOK % x.id, #'cachekey': CACHEKEY_CASHBOOK % x.id,
}, { }, {
'model': 'currency.currency.rate', 'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)], 'query': [('currency.id', '=', x.currency.id)],
@ -392,9 +401,15 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
} }
# read from cache # read from cache
logger.warning('## get_balance_cashbook-KEYS %(time)s' % {
'time': datetime.now().isoformat(),
})
(todo_cashbook, result) = MemCache.read_from_cache( (todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result) cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0: if len(todo_cashbook) == 0:
logger.warning('## get_balance_cashbook-HIT %(time)s' % {
'time': datetime.now().isoformat(),
})
return result return result
# query balances of cashbooks and sub-cashbooks # query balances of cashbooks and sub-cashbooks
@ -430,7 +445,10 @@ class Book(tree(separator='/'), Workflow, ModelSQL, ModelView):
result['balance_ref'][record[0]] += Currency.compute( result['balance_ref'][record[0]] += Currency.compute(
record[2], record[5], record[3]) record[2], record[5], record[3])
MemCache.store_result(cashbooks, cache_keys, result) MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook)
logger.warning('## get_balance_cashbook-END %(time)s' % {
'time': datetime.now().isoformat(),
})
return result return result
@fields.depends('btype') @fields.depends('btype')

22
line.py
View file

@ -1008,9 +1008,9 @@ class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView):
)) ))
records = super(Line, cls).create(vlist) records = super(Line, cls).create(vlist)
for record in records: # ~ for record in records:
for x in record.cashbook.get_cachekeys_by_hierarchy(): # ~ for x in record.cashbook.get_cachekeys_by_hierarchy():
MemCache.record_update(x, record) # ~ MemCache.record_update(x, record)
return records return records
@classmethod @classmethod
@ -1066,11 +1066,11 @@ class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView):
super(Line, cls).write(*to_write) super(Line, cls).write(*to_write)
actions = iter(to_write) # ~ actions = iter(to_write)
for records, values in zip(actions, actions): # ~ for records, values in zip(actions, actions):
for record in records: # ~ for record in records:
for x in record.cashbook.get_cachekeys_by_hierarchy(): # ~ for x in record.cashbook.get_cachekeys_by_hierarchy():
MemCache.record_update(x, record) # ~ MemCache.record_update(x, record)
@classmethod @classmethod
def delete(cls, lines): def delete(cls, lines):
@ -1079,9 +1079,9 @@ class Line(SecondCurrencyMixin, MemCacheIndexMx, Workflow, ModelSQL, ModelView):
MemCache = Pool().get('cashbook.memcache') MemCache = Pool().get('cashbook.memcache')
cls.check_permission_delete(lines) cls.check_permission_delete(lines)
for line in lines: # ~ for line in lines:
for x in line.cashbook.get_cachekeys_by_hierarchy(): # ~ for x in line.cashbook.get_cachekeys_by_hierarchy():
MemCache.record_update(x, None) # ~ MemCache.record_update(x, None)
super(Line, cls).delete(lines) super(Line, cls).delete(lines)
# end Line # end Line

44
model.py
View file

@ -7,12 +7,14 @@ from trytond.model import MultiValueMixin, ValueMixin, fields, Unique, Model
from trytond.transaction import Transaction from trytond.transaction import Transaction
from trytond.pool import Pool from trytond.pool import Pool
from trytond.cache import MemoryCache from trytond.cache import MemoryCache
from datetime import timedelta from datetime import timedelta, datetime
from decimal import Decimal from decimal import Decimal
from sql import With, Literal from sql import With, Literal
from sql.functions import Function from sql.functions import Function
from sql.conditionals import Coalesce from sql.conditionals import Coalesce
import copy import copy, logging
logger = logging.getLogger(__name__)
ENABLE_CACHE = True ENABLE_CACHE = True
@ -91,19 +93,26 @@ class MemCache(Model):
return copy.deepcopy(cls._cashbook_value_cache.get(cache_key)) return copy.deepcopy(cls._cashbook_value_cache.get(cache_key))
@classmethod @classmethod
def store_result(cls, records, cache_keys, values): def store_result(cls, records, cache_keys, values, skip_records=[]):
""" store result to cache """ store result to cache
""" """
if ENABLE_CACHE == False: if ENABLE_CACHE == False:
return return
for record in records: for record in records:
if record not in skip_records:
continue
data = {x:values[x][record.id] data = {x:values[x][record.id]
for x in values.keys() for x in values.keys()
if record.id in values[x].keys()} if record.id in values[x].keys()}
cls._cashbook_value_cache.set(cache_keys[record.id], copy.deepcopy(data)) cls._cashbook_value_cache.set(cache_keys[record.id], copy.deepcopy(data))
print('\n## SYNC-GO-1') # ~ logger.info('memcache-VALUE-SET %(time)s key=%(key)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ 'key': cache_keys[record.id],
# ~ })
cls._cashbook_value_cache.sync(Transaction()) cls._cashbook_value_cache.sync(Transaction())
print('-- SYNC-FERTIG-1') # ~ logger.info('memcache-VALUE-SYNC %(time)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ })
@classmethod @classmethod
def store_value(cls, cache_key, values): def store_value(cls, cache_key, values):
@ -112,9 +121,10 @@ class MemCache(Model):
if ENABLE_CACHE == False: if ENABLE_CACHE == False:
return return
cls._cashbook_value_cache.set(cache_key, copy.deepcopy(values)) cls._cashbook_value_cache.set(cache_key, copy.deepcopy(values))
print('\n## SYNC-GO-2') # ~ logger.info('memcache-VALUE-SET %(time)s key=%(key)s' % {
cls._cashbook_value_cache.sync(Transaction()) # ~ 'time': datetime.now().isoformat(),
print('-- SYNC-FERTIG-2') # ~ 'key': cache_key,
# ~ })
@classmethod @classmethod
def read_from_cache(cls, records, cache_keys, names, result): def read_from_cache(cls, records, cache_keys, names, result):
@ -135,7 +145,15 @@ class MemCache(Model):
if result[name][record.id] is None: if result[name][record.id] is None:
result[name][record.id] = Decimal('0.0') result[name][record.id] = Decimal('0.0')
result[name][record.id] += values[name] result[name][record.id] += values[name]
# ~ logger.info('memcache-VALUE-READ-HIT %(time)s key=%(key)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ 'key': cache_keys[record.id],
# ~ })
else : else :
# ~ logger.info('memcache-VALUE-READ-FAIL %(time)s key=%(key)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ 'key': cache_keys[record.id],
# ~ })
todo_records.append(record) todo_records.append(record)
return (todo_records, result) return (todo_records, result)
@ -149,6 +167,11 @@ class MemCache(Model):
if ENABLE_CACHE == False: if ENABLE_CACHE == False:
return '-' return '-'
# ~ logger.info('memcache-KEY %(time)s name=%(name)s record=%(record)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ 'name': name,
# ~ 'record': str(record),
# ~ })
fname = [name, str(record.id)] fname = [name, str(record.id)]
fname.extend(addkeys) fname.extend(addkeys)
@ -192,6 +215,11 @@ class MemCache(Model):
if 'cachekey' in line.keys(): if 'cachekey' in line.keys():
key = cls.store_value(line['cachekey'], fname[-1]) key = cls.store_value(line['cachekey'], fname[-1])
# ~ logger.info('memcache-KEY-RESULT %(time)s name=%(name)s result=%(result)s' % {
# ~ 'time': datetime.now().isoformat(),
# ~ 'name': name,
# ~ 'result': '-'.join(fname),
# ~ })
return '-'.join(fname) return '-'.join(fname)
@classmethod @classmethod

10
tests/test_book.py
View file

@ -190,11 +190,11 @@ class BookTestCase(ModuleTestCase):
self.assertEqual(book.rec_name, 'Level 1') self.assertEqual(book.rec_name, 'Level 1')
self.assertEqual(len(book.childs), 1) self.assertEqual(len(book.childs), 1)
self.assertEqual(book.childs[0].rec_name, 'Level 1/Level 2 | 0.00 usd | Open') self.assertEqual(book.childs[0].rec_name, 'Level 1/Level 2 | 0.00 usd | Open')
self.assertEqual(book.get_cachekeys_by_hierarchy(), [CACHEKEY_CASHBOOK % book.id]) # ~ self.assertEqual(book.get_cachekeys_by_hierarchy(), [CACHEKEY_CASHBOOK % book.id])
self.assertEqual(book.childs[0].get_cachekeys_by_hierarchy(), [ # ~ self.assertEqual(book.childs[0].get_cachekeys_by_hierarchy(), [
CACHEKEY_CASHBOOK % book.id, # ~ CACHEKEY_CASHBOOK % book.id,
CACHEKEY_CASHBOOK % book.childs[0].id # ~ CACHEKEY_CASHBOOK % book.childs[0].id
]) # ~ ])
@with_transaction() @with_transaction()
def test_book_deny_delete_open(self): def test_book_deny_delete_open(self):