From da7fc9cb655b7e8fe613fc9d53887b7b01cac58c Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sat, 4 Mar 2023 21:21:54 +0100 Subject: [PATCH] book: optimize query for speed --- book.py | 256 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 158 insertions(+), 98 deletions(-) diff --git a/book.py b/book.py index 68139e1..a943ddb 100644 --- a/book.py +++ b/book.py @@ -10,14 +10,18 @@ from trytond.modules.cashbook.book import STATES2, DEPENDS2 from trytond.transaction import Transaction from trytond.report import Report from decimal import Decimal -from datetime import timedelta +from datetime import timedelta, datetime from sql import Literal from sql.functions import CurrentDate from sql.aggregate import Sum from sql.conditionals import Case, Coalesce -from trytond.modules.cashbook.model import CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK +from trytond.modules.cashbook.model import sub_ids_hierarchical,\ + CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK, AnyInArray from .asset import CACHEKEY_ASSETRATE +import logging +logger = logging.getLogger(__name__) + class Book(SymbolMixin, metaclass=PoolMeta): __name__ = 'cashbook.book' @@ -224,10 +228,12 @@ class Book(SymbolMixin, metaclass=PoolMeta): """ pool = Pool() Line = pool.get('cashbook.line') + Currency = pool.get('currency.currency') (tab_line1, tab_line_yield) = Line.get_yield_data_sql() (tab_line2, tab_line_gainloss) = Line.get_gainloss_data_sql() tab_book = cls.__table__() tab_line = Line.__table__() + tab_cur = Currency.__table__() where = Literal(True) if date_from: @@ -237,6 +243,8 @@ class Book(SymbolMixin, metaclass=PoolMeta): query = tab_book.join(tab_line, condition=tab_line.cashbook==tab_book.id, + ).join(tab_cur, + condition=tab_cur.id==tab_book.currency, ).join(tab_line_yield, condition=tab_line_yield.id==tab_line.id, ).join(tab_line_gainloss, @@ -246,7 +254,8 @@ class Book(SymbolMixin, metaclass=PoolMeta): Sum(tab_line_yield.fee).as_('fee'), Sum(tab_line_yield.dividend).as_('dividend'), Sum(tab_line_gainloss.gainloss).as_('gainloss'), - group_by=[tab_book.id], + tab_cur.digits.as_('currency_digits'), + group_by=[tab_book.id, tab_cur.digits], where=where ) return (tab_book, query) @@ -267,12 +276,18 @@ class Book(SymbolMixin, metaclass=PoolMeta): 'yield_sales', 'yield_fee_12m', 'yield_dividend_12m', 'yield_sales_12m']} - def quantize_val(value, line): + logger.warning('## get_yield_data-GO %(time)s %(ids)s %(name)s' % { + 'time': datetime.now().isoformat(), + 'ids': str([x.id for x in cashbooks]), + 'name': names, + }) + + def quantize_val(value, digits): """ quantize... """ return ( value or Decimal('0.0') - ).quantize(Decimal(str(1/10**line.currency_digits))) + ).quantize(Decimal(str(1/10 ** digits))) query_date = context.get('date', IrDate.today()) cache_keys = { @@ -282,7 +297,7 @@ class Book(SymbolMixin, metaclass=PoolMeta): query = [{ 'model': 'cashbook.line', 'query': [('cashbook.parent', 'child_of', [x.id])], - 'cachekey': CACHEKEY_CASHBOOK % x.id, + #'cachekey': CACHEKEY_CASHBOOK % x.id, }, { 'model': 'currency.currency.rate', 'query': [('currency.id', '=', x.currency.id)], @@ -290,16 +305,22 @@ class Book(SymbolMixin, metaclass=PoolMeta): }, { 'model': 'investment.rate', 'query': [('asset.id', '=', x.asset.id)], - 'cachekey': CACHEKEY_ASSETRATE % x.asset.id, + #'cachekey': CACHEKEY_ASSETRATE % x.asset.id, } if x.asset is not None else {}], addkeys = [query_date.isoformat()]) for x in cashbooks } + logger.warning('## get_yield_data-KEYS %(time)s' % { + 'time': datetime.now().isoformat(), + }) # read from cache (todo_cashbook, result) = MemCache.read_from_cache( cashbooks, cache_keys, names, result) if len(todo_cashbook) == 0: + logger.warning('## get_yield_data-HIT %(time)s' % { + 'time': datetime.now().isoformat(), + }) return result # results for 'total' @@ -321,19 +342,20 @@ class Book(SymbolMixin, metaclass=PoolMeta): records_12m = cursor.fetchall() for record in records_total: - book = CashBook(record[0]) - result['yield_fee_total'][record[0]] = quantize_val(record[1], book) - result['yield_dividend_total'][record[0]] = quantize_val(record[2], book) - result['yield_sales'][record[0]] = quantize_val(record[3], book) + result['yield_fee_total'][record[0]] = quantize_val(record[1], record[4]) + result['yield_dividend_total'][record[0]] = quantize_val(record[2], record[4]) + result['yield_sales'][record[0]] = quantize_val(record[3], record[4]) for record in records_12m: - book = CashBook(record[0]) - result['yield_fee_12m'][record[0]] = quantize_val(record[1], book) - result['yield_dividend_12m'][record[0]] = quantize_val(record[2], book) - result['yield_sales_12m'][record[0]] = quantize_val(record[3], book) + result['yield_fee_12m'][record[0]] = quantize_val(record[1], record[4]) + result['yield_dividend_12m'][record[0]] = quantize_val(record[2], record[4]) + result['yield_sales_12m'][record[0]] = quantize_val(record[3], record[4]) # store to cache - MemCache.store_result(cashbooks, cache_keys, result) + MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook) + logger.warning('## get_yield_data-END %(time)s' % { + 'time': datetime.now().isoformat(), + }) return {x:result[x] for x in names} @classmethod @@ -387,10 +409,9 @@ class Book(SymbolMixin, metaclass=PoolMeta): tab_asset.uom, # 6 tab_book.quantity_uom, # 7 tab_asset.currency.as_('asset_currency'), #8 - Coalesce(tab_balance.balance, Decimal('0.0')).as_('balance'), #9 ( Sum(query_yield.fee) + tab_balance.balance - ).as_('purchase_amount'), #10 + ).as_('purchase_amount'), #9 group_by=[tab_book.id, tab_rate.rate, tab_book.currency, tab_cur.digits, tab_asset.uom, tab_book.quantity_uom, tab_asset.currency, @@ -402,6 +423,9 @@ class Book(SymbolMixin, metaclass=PoolMeta): @classmethod def get_asset_quantity(cls, cashbooks, names): """ get quantities + field: quantity, quantity_all, current_value, + current_value_ref, diff_amount, diff_percent, + current_rate, purchase_amount """ pool = Pool() CBook = pool.get('cashbook.book') @@ -414,36 +438,54 @@ class Book(SymbolMixin, metaclass=PoolMeta): (query, tab_book) = cls.get_asset_quantity_sql() company_currency = CBook.default_currency() - result = {x:{y.id: None for y in cashbooks} for x in names} + result = { + x:{y.id: None for y in cashbooks} + for x in ['quantity', 'quantity_all', 'current_value', + 'current_value_ref', 'diff_amount', 'diff_percent', + 'current_rate', 'purchase_amount', 'purchase_amount_ref', + 'digits'] + } + + logger.warning('## get_asset_quantity-GO %(time)s %(ids)s %(name)s' % { + 'time': datetime.now().isoformat(), + 'ids': str([x.id for x in cashbooks]), + 'name': names, + }) cache_keys = { - x.id: MemCache.get_key_by_record( - name = 'get_asset_quantity', - record = x, - query = [{ - 'model': 'cashbook.line', - 'query': [('cashbook.parent', 'child_of', [x.id])], - 'cachekey': CACHEKEY_CASHBOOK % x.id, - }, { - 'model': 'currency.currency.rate', - 'query': [('currency.id', '=', x.currency.id)], - 'cachekey': CACHEKEY_CURRENCY % x.currency.id, - }, { - 'model': 'investment.rate', - 'query': [('asset.id', '=', x.asset.id)], - 'cachekey': CACHEKEY_ASSETRATE % x.asset.id, - } if x.asset is not None else {}], - addkeys=[ - str(company_currency), - str(context.get('qdate', IrDate.today()).toordinal()), - ]) - for x in cashbooks + x.id: MemCache.get_key_by_record( + name = 'get_asset_quantity', + record = x, + query = [{ + 'model': 'cashbook.line', + 'query': [('cashbook.parent', 'child_of', [x.id])], + #'cachekey': CACHEKEY_CASHBOOK % x.id, + }, { + 'model': 'currency.currency.rate', + 'query': [('currency.id', '=', x.currency.id)], + 'cachekey': CACHEKEY_CURRENCY % x.currency.id, + }, { + 'model': 'investment.rate', + 'query': [('asset.id', '=', x.asset.id)], + #'cachekey': CACHEKEY_ASSETRATE % x.asset.id, + } if x.asset is not None else {}], + addkeys=[ + str(company_currency), + str(context.get('qdate', IrDate.today()).toordinal()), + ]) + for x in cashbooks } # read from cache + logger.warning('## get_asset_quantity-KEYS %(time)s' % { + 'time': datetime.now().isoformat(), + }) (todo_cashbook, result) = MemCache.read_from_cache( cashbooks, cache_keys, names, result) if len(todo_cashbook) == 0: + logger.warning('## get_asset_quantity-HIT %(time)s' % { + 'time': datetime.now().isoformat(), + }) return result def values_from_record(rdata): @@ -471,25 +513,24 @@ class Book(SymbolMixin, metaclass=PoolMeta): rdata[3] * rdata[1] / uom_factor, company_currency if company_currency is not None else rdata[8], ), - 'diff_amount': current_value - rdata[10], + 'diff_amount': current_value - rdata[9], 'diff_percent': ( Decimal('100.0') * current_value / \ - rdata[10] - Decimal('100.0') + rdata[9] - Decimal('100.0') ).quantize(Decimal(str(1/10**rdata[5]))) \ - if rdata[10] != Decimal('0.0') else None, + if rdata[9] != Decimal('0.0') else None, 'current_rate': ( current_value / rdata[1] ).quantize(Decimal(str(1/10**rdata[5]))) \ if rdata[1] != Decimal('0.0') else None, - 'purchase_amount': record[10].quantize(Decimal(str(1/10**rdata[5]))), + 'purchase_amount': record[9].quantize(Decimal(str(1/10**rdata[5]))), 'purchase_amount_ref': Currency.compute( rdata[4], - record[10], + record[9], company_currency if company_currency is not None else rdata[4], ), }) - result_cache = {} ids_assetbooks = [x.id for x in cashbooks if x.btype is not None] ids_nonebtypes = [x.id for x in cashbooks if x.btype is None] @@ -501,80 +542,99 @@ class Book(SymbolMixin, metaclass=PoolMeta): for record in records: (book_id, values) = values_from_record(record) - result_cache[book_id] = values - result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref for name in values.keys(): - if name not in result.keys(): - result[name] = {} result[name][book_id] = values[name] + logger.warning('## get_asset_quantity-PART1 %(time)s' % { + 'time': datetime.now().isoformat(), + }) + # add aggregated values of cashbooks without type aggr_names = ['current_value', 'current_value_ref', 'diff_amount', 'diff_percent'] queried_names = list(set(aggr_names).intersection(set(names))) - if len(queried_names) > 0: - # query all subordered asset-cashbooks for - # btype=None-cashbooks - query1 = [('btype.feature', '=', 'asset'), - ('parent', 'child_of', ids_nonebtypes)] - if len(result_cache.keys()) > 0: - query1.append(('id', 'not in', result_cache.keys())) - books_query = CBook.search(query1, query=True) + if (len(queried_names) > 0) and (len(ids_nonebtypes) > 0): + # query all subordered asset-cashbooks to get values for + # cashbooks without type - # add results to cache - (query, tab_book) = cls.get_asset_quantity_sql() - query.where &= tab_book.id.in_(books_query) + (tab_quantity, tab_book) = cls.get_asset_quantity_sql() + tab_subids = sub_ids_hierarchical('cashbook.book') + query = tab_book.join(tab_subids, + condition=tab_book.id==tab_subids.parent, + ).join(tab_quantity, + condition=tab_quantity.id==AnyInArray(tab_subids.subids), + ).select( + tab_book.id, + Sum(tab_quantity.quantity), # 1 + Sum(tab_quantity.quantity_all), # 2 + tab_quantity.rate, # 3 + tab_quantity.currency, # 4 + tab_quantity.currency_digits, # 5 + tab_quantity.uom, # 6 + tab_quantity.quantity_uom, # 7 + tab_quantity.asset_currency, # 8 + Sum(tab_quantity.purchase_amount), # 9 + tab_book.currency.as_('currency_book'), # 10 + where=tab_book.id.in_(ids_nonebtypes), + group_by=[tab_book.id, tab_quantity.rate, + tab_quantity.currency, tab_quantity.currency_digits, + tab_quantity.uom, tab_quantity.quantity_uom, + tab_quantity.asset_currency,tab_book.currency], + ) cursor.execute(*query) records = cursor.fetchall() for record in records: (book_id, values) = values_from_record(record) - result_cache[book_id] = values - result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref - # aggregate sub-cashbooks to requested cashbooks from cache - for id_none in ids_nonebtypes: - records = CBook.search([ - ('btype.feature', '=', 'asset'), - ('parent', 'child_of', [id_none]) - ]) + for name in ['current_value', 'diff_amount', + 'current_value_ref', 'purchase_amount_ref']: + if result[name][book_id] is None: + result[name][book_id] = Decimal('0.0') - values = {x:Decimal('0.0') for x in aggr_names+['purchase_amount_ref']} - for record in records: - for name in aggr_names+['purchase_amount_ref']: - values[name] += \ - result_cache.get(record.id, {}).get(name, Decimal('0.0')) + value = Decimal('0.0') + if name == 'current_value': + value = Currency.compute( + company_currency if company_currency is not None else record[4], + values['current_value_ref'], + record[10], + ) + elif name == 'diff_amount': + value = Currency.compute( + company_currency if company_currency is not None else record[4], + values['current_value_ref'] - values['purchase_amount_ref'], + record[10], + ) + elif name in ['current_value_ref', 'purchase_amount_ref']: + value = values[name] + result['digits'][book_id] = record[5] + result[name][book_id] += value - # convert current-value-ref in company-currency to - # currency of current cashbook - cbook = CBook(id_none) - values['current_value'] = Currency.compute( - company_currency if company_currency is not None else cbook.currency, - values['current_value_ref'], - cbook.currency, - ) + # diff_prcent + for id_book in ids_nonebtypes: + c_val = result['current_value_ref'][id_book] + p_amount = result['purchase_amount_ref'][id_book] + digits = result['digits'][id_book] - values['diff_amount'] = Currency.compute( - company_currency if company_currency is not None else cbook.currency, - values['current_value_ref'] - values['purchase_amount_ref'], - cbook.currency, - ) + if (p_amount == Decimal('0.0')) or (p_amount is None) or (c_val is None): + continue - values['diff_percent'] = \ - (Decimal('100.0') * values['current_value_ref'] / \ - values['purchase_amount_ref'] - Decimal('100.0') - ).quantize( - Decimal(str(1/10**cbook.currency_digits)) - ) if values['purchase_amount_ref'] != Decimal('0.0') else None - for name in values.keys(): - if name not in result.keys(): - result[name] = {} - result[name][id_none] = values[name] + result['diff_percent'][id_book] = ( + Decimal('100.0') * c_val / p_amount - Decimal('100.0') + ).quantize(Decimal(str(1/10 ** digits))) + result['digits'][id_book] = None + + logger.warning('## get_asset_quantity-PART2 %(time)s' % { + 'time': datetime.now().isoformat(), + }) # store to cache - MemCache.store_result(cashbooks, cache_keys, result) + MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook) + logger.warning('## get_asset_quantity-END %(time)s' % { + 'time': datetime.now().isoformat(), + }) return {x:result[x] for x in names} @fields.depends('id')