mdsfred/cashbook_investment
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

book: optimize query for speed

Browse source
This commit is contained in:
Frederik Jaeckel 2023-03-04 21:21:54 +01:00
parent e244e75d8a
commit da7fc9cb65
1 changed files with 158 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

214
book.py
View file

@ -10,14 +10,18 @@ from trytond.modules.cashbook.book import STATES2, DEPENDS2
from trytond.transaction import Transaction from trytond.transaction import Transaction
from trytond.report import Report from trytond.report import Report
from decimal import Decimal from decimal import Decimal
from datetime import timedelta from datetime import timedelta, datetime
from sql import Literal from sql import Literal
from sql.functions import CurrentDate from sql.functions import CurrentDate
from sql.aggregate import Sum from sql.aggregate import Sum
from sql.conditionals import Case, Coalesce from sql.conditionals import Case, Coalesce
from trytond.modules.cashbook.model import CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK from trytond.modules.cashbook.model import sub_ids_hierarchical,\
CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK, AnyInArray
from .asset import CACHEKEY_ASSETRATE from .asset import CACHEKEY_ASSETRATE
import logging
logger = logging.getLogger(__name__)
class Book(SymbolMixin, metaclass=PoolMeta): class Book(SymbolMixin, metaclass=PoolMeta):
__name__ = 'cashbook.book' __name__ = 'cashbook.book'
@ -224,10 +228,12 @@ class Book(SymbolMixin, metaclass=PoolMeta):
""" """
pool = Pool() pool = Pool()
Line = pool.get('cashbook.line') Line = pool.get('cashbook.line')
Currency = pool.get('currency.currency')
(tab_line1, tab_line_yield) = Line.get_yield_data_sql() (tab_line1, tab_line_yield) = Line.get_yield_data_sql()
(tab_line2, tab_line_gainloss) = Line.get_gainloss_data_sql() (tab_line2, tab_line_gainloss) = Line.get_gainloss_data_sql()
tab_book = cls.__table__() tab_book = cls.__table__()
tab_line = Line.__table__() tab_line = Line.__table__()
tab_cur = Currency.__table__()
where = Literal(True) where = Literal(True)
if date_from: if date_from:
@ -237,6 +243,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
query = tab_book.join(tab_line, query = tab_book.join(tab_line,
condition=tab_line.cashbook==tab_book.id, condition=tab_line.cashbook==tab_book.id,
).join(tab_cur,
condition=tab_cur.id==tab_book.currency,
).join(tab_line_yield, ).join(tab_line_yield,
condition=tab_line_yield.id==tab_line.id, condition=tab_line_yield.id==tab_line.id,
).join(tab_line_gainloss, ).join(tab_line_gainloss,
@ -246,7 +254,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
Sum(tab_line_yield.fee).as_('fee'), Sum(tab_line_yield.fee).as_('fee'),
Sum(tab_line_yield.dividend).as_('dividend'), Sum(tab_line_yield.dividend).as_('dividend'),
Sum(tab_line_gainloss.gainloss).as_('gainloss'), Sum(tab_line_gainloss.gainloss).as_('gainloss'),
group_by=[tab_book.id], tab_cur.digits.as_('currency_digits'),
group_by=[tab_book.id, tab_cur.digits],
where=where where=where
) )
return (tab_book, query) return (tab_book, query)
@ -267,12 +276,18 @@ class Book(SymbolMixin, metaclass=PoolMeta):
'yield_sales', 'yield_fee_12m', 'yield_dividend_12m', 'yield_sales', 'yield_fee_12m', 'yield_dividend_12m',
'yield_sales_12m']} 'yield_sales_12m']}
def quantize_val(value, line): logger.warning('## get_yield_data-GO %(time)s %(ids)s %(name)s' % {
'time': datetime.now().isoformat(),
'ids': str([x.id for x in cashbooks]),
'name': names,
})
def quantize_val(value, digits):
""" quantize... """ quantize...
""" """
return ( return (
value or Decimal('0.0') value or Decimal('0.0')
).quantize(Decimal(str(1/10**line.currency_digits))) ).quantize(Decimal(str(1/10 ** digits)))
query_date = context.get('date', IrDate.today()) query_date = context.get('date', IrDate.today())
cache_keys = { cache_keys = {
@ -282,7 +297,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
query = [{ query = [{
'model': 'cashbook.line', 'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])], 'query': [('cashbook.parent', 'child_of', [x.id])],
'cachekey': CACHEKEY_CASHBOOK % x.id, #'cachekey': CACHEKEY_CASHBOOK % x.id,
}, { }, {
'model': 'currency.currency.rate', 'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)], 'query': [('currency.id', '=', x.currency.id)],
@ -290,16 +305,22 @@ class Book(SymbolMixin, metaclass=PoolMeta):
}, { }, {
'model': 'investment.rate', 'model': 'investment.rate',
'query': [('asset.id', '=', x.asset.id)], 'query': [('asset.id', '=', x.asset.id)],
'cachekey': CACHEKEY_ASSETRATE % x.asset.id, #'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
} if x.asset is not None else {}], } if x.asset is not None else {}],
addkeys = [query_date.isoformat()]) addkeys = [query_date.isoformat()])
for x in cashbooks for x in cashbooks
} }
logger.warning('## get_yield_data-KEYS %(time)s' % {
'time': datetime.now().isoformat(),
})
# read from cache # read from cache
(todo_cashbook, result) = MemCache.read_from_cache( (todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result) cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0: if len(todo_cashbook) == 0:
logger.warning('## get_yield_data-HIT %(time)s' % {
'time': datetime.now().isoformat(),
})
return result return result
# results for 'total' # results for 'total'
@ -321,19 +342,20 @@ class Book(SymbolMixin, metaclass=PoolMeta):
records_12m = cursor.fetchall() records_12m = cursor.fetchall()
for record in records_total: for record in records_total:
book = CashBook(record[0]) result['yield_fee_total'][record[0]] = quantize_val(record[1], record[4])
result['yield_fee_total'][record[0]] = quantize_val(record[1], book) result['yield_dividend_total'][record[0]] = quantize_val(record[2], record[4])
result['yield_dividend_total'][record[0]] = quantize_val(record[2], book) result['yield_sales'][record[0]] = quantize_val(record[3], record[4])
result['yield_sales'][record[0]] = quantize_val(record[3], book)
for record in records_12m: for record in records_12m:
book = CashBook(record[0]) result['yield_fee_12m'][record[0]] = quantize_val(record[1], record[4])
result['yield_fee_12m'][record[0]] = quantize_val(record[1], book) result['yield_dividend_12m'][record[0]] = quantize_val(record[2], record[4])
result['yield_dividend_12m'][record[0]] = quantize_val(record[2], book) result['yield_sales_12m'][record[0]] = quantize_val(record[3], record[4])
result['yield_sales_12m'][record[0]] = quantize_val(record[3], book)
# store to cache # store to cache
MemCache.store_result(cashbooks, cache_keys, result) MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook)
logger.warning('## get_yield_data-END %(time)s' % {
'time': datetime.now().isoformat(),
})
return {x:result[x] for x in names} return {x:result[x] for x in names}
@classmethod @classmethod
@ -387,10 +409,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
tab_asset.uom, # 6 tab_asset.uom, # 6
tab_book.quantity_uom, # 7 tab_book.quantity_uom, # 7
tab_asset.currency.as_('asset_currency'), #8 tab_asset.currency.as_('asset_currency'), #8
Coalesce(tab_balance.balance, Decimal('0.0')).as_('balance'), #9
( (
Sum(query_yield.fee) + tab_balance.balance Sum(query_yield.fee) + tab_balance.balance
).as_('purchase_amount'), #10 ).as_('purchase_amount'), #9
group_by=[tab_book.id, tab_rate.rate, group_by=[tab_book.id, tab_rate.rate,
tab_book.currency, tab_cur.digits, tab_asset.uom, tab_book.currency, tab_cur.digits, tab_asset.uom,
tab_book.quantity_uom, tab_asset.currency, tab_book.quantity_uom, tab_asset.currency,
@ -402,6 +423,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
@classmethod @classmethod
def get_asset_quantity(cls, cashbooks, names): def get_asset_quantity(cls, cashbooks, names):
""" get quantities """ get quantities
field: quantity, quantity_all, current_value,
current_value_ref, diff_amount, diff_percent,
current_rate, purchase_amount
""" """
pool = Pool() pool = Pool()
CBook = pool.get('cashbook.book') CBook = pool.get('cashbook.book')
@ -414,7 +438,19 @@ class Book(SymbolMixin, metaclass=PoolMeta):
(query, tab_book) = cls.get_asset_quantity_sql() (query, tab_book) = cls.get_asset_quantity_sql()
company_currency = CBook.default_currency() company_currency = CBook.default_currency()
result = {x:{y.id: None for y in cashbooks} for x in names} result = {
x:{y.id: None for y in cashbooks}
for x in ['quantity', 'quantity_all', 'current_value',
'current_value_ref', 'diff_amount', 'diff_percent',
'current_rate', 'purchase_amount', 'purchase_amount_ref',
'digits']
}
logger.warning('## get_asset_quantity-GO %(time)s %(ids)s %(name)s' % {
'time': datetime.now().isoformat(),
'ids': str([x.id for x in cashbooks]),
'name': names,
})
cache_keys = { cache_keys = {
x.id: MemCache.get_key_by_record( x.id: MemCache.get_key_by_record(
@ -423,7 +459,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
query = [{ query = [{
'model': 'cashbook.line', 'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])], 'query': [('cashbook.parent', 'child_of', [x.id])],
'cachekey': CACHEKEY_CASHBOOK % x.id, #'cachekey': CACHEKEY_CASHBOOK % x.id,
}, { }, {
'model': 'currency.currency.rate', 'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)], 'query': [('currency.id', '=', x.currency.id)],
@ -431,7 +467,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
}, { }, {
'model': 'investment.rate', 'model': 'investment.rate',
'query': [('asset.id', '=', x.asset.id)], 'query': [('asset.id', '=', x.asset.id)],
'cachekey': CACHEKEY_ASSETRATE % x.asset.id, #'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
} if x.asset is not None else {}], } if x.asset is not None else {}],
addkeys=[ addkeys=[
str(company_currency), str(company_currency),
@ -441,9 +477,15 @@ class Book(SymbolMixin, metaclass=PoolMeta):
} }
# read from cache # read from cache
logger.warning('## get_asset_quantity-KEYS %(time)s' % {
'time': datetime.now().isoformat(),
})
(todo_cashbook, result) = MemCache.read_from_cache( (todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result) cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0: if len(todo_cashbook) == 0:
logger.warning('## get_asset_quantity-HIT %(time)s' % {
'time': datetime.now().isoformat(),
})
return result return result
def values_from_record(rdata): def values_from_record(rdata):
@ -471,25 +513,24 @@ class Book(SymbolMixin, metaclass=PoolMeta):
rdata[3] * rdata[1] / uom_factor, rdata[3] * rdata[1] / uom_factor,
company_currency if company_currency is not None else rdata[8], company_currency if company_currency is not None else rdata[8],
), ),
'diff_amount': current_value - rdata[10], 'diff_amount': current_value - rdata[9],
'diff_percent': ( 'diff_percent': (
Decimal('100.0') * current_value / \ Decimal('100.0') * current_value / \
rdata[10] - Decimal('100.0') rdata[9] - Decimal('100.0')
).quantize(Decimal(str(1/10**rdata[5]))) \ ).quantize(Decimal(str(1/10**rdata[5]))) \
if rdata[10] != Decimal('0.0') else None, if rdata[9] != Decimal('0.0') else None,
'current_rate': ( 'current_rate': (
current_value / rdata[1] current_value / rdata[1]
).quantize(Decimal(str(1/10**rdata[5]))) \ ).quantize(Decimal(str(1/10**rdata[5]))) \
if rdata[1] != Decimal('0.0') else None, if rdata[1] != Decimal('0.0') else None,
'purchase_amount': record[10].quantize(Decimal(str(1/10**rdata[5]))), 'purchase_amount': record[9].quantize(Decimal(str(1/10**rdata[5]))),
'purchase_amount_ref': Currency.compute( 'purchase_amount_ref': Currency.compute(
rdata[4], rdata[4],
record[10], record[9],
company_currency if company_currency is not None else rdata[4], company_currency if company_currency is not None else rdata[4],
), ),
}) })
result_cache = {}
ids_assetbooks = [x.id for x in cashbooks if x.btype is not None] ids_assetbooks = [x.id for x in cashbooks if x.btype is not None]
ids_nonebtypes = [x.id for x in cashbooks if x.btype is None] ids_nonebtypes = [x.id for x in cashbooks if x.btype is None]
@ -501,80 +542,99 @@ class Book(SymbolMixin, metaclass=PoolMeta):
for record in records: for record in records:
(book_id, values) = values_from_record(record) (book_id, values) = values_from_record(record)
result_cache[book_id] = values
result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref
for name in values.keys(): for name in values.keys():
if name not in result.keys():
result[name] = {}
result[name][book_id] = values[name] result[name][book_id] = values[name]
logger.warning('## get_asset_quantity-PART1 %(time)s' % {
'time': datetime.now().isoformat(),
})
# add aggregated values of cashbooks without type # add aggregated values of cashbooks without type
aggr_names = ['current_value', 'current_value_ref', aggr_names = ['current_value', 'current_value_ref',
'diff_amount', 'diff_percent'] 'diff_amount', 'diff_percent']
queried_names = list(set(aggr_names).intersection(set(names))) queried_names = list(set(aggr_names).intersection(set(names)))
if len(queried_names) > 0: if (len(queried_names) > 0) and (len(ids_nonebtypes) > 0):
# query all subordered asset-cashbooks for # query all subordered asset-cashbooks to get values for
# btype=None-cashbooks # cashbooks without type
query1 = [('btype.feature', '=', 'asset'),
('parent', 'child_of', ids_nonebtypes)]
if len(result_cache.keys()) > 0:
query1.append(('id', 'not in', result_cache.keys()))
books_query = CBook.search(query1, query=True)
# add results to cache (tab_quantity, tab_book) = cls.get_asset_quantity_sql()
(query, tab_book) = cls.get_asset_quantity_sql() tab_subids = sub_ids_hierarchical('cashbook.book')
query.where &= tab_book.id.in_(books_query) query = tab_book.join(tab_subids,
condition=tab_book.id==tab_subids.parent,
).join(tab_quantity,
condition=tab_quantity.id==AnyInArray(tab_subids.subids),
).select(
tab_book.id,
Sum(tab_quantity.quantity), # 1
Sum(tab_quantity.quantity_all), # 2
tab_quantity.rate, # 3
tab_quantity.currency, # 4
tab_quantity.currency_digits, # 5
tab_quantity.uom, # 6
tab_quantity.quantity_uom, # 7
tab_quantity.asset_currency, # 8
Sum(tab_quantity.purchase_amount), # 9
tab_book.currency.as_('currency_book'), # 10
where=tab_book.id.in_(ids_nonebtypes),
group_by=[tab_book.id, tab_quantity.rate,
tab_quantity.currency, tab_quantity.currency_digits,
tab_quantity.uom, tab_quantity.quantity_uom,
tab_quantity.asset_currency,tab_book.currency],
)
cursor.execute(*query) cursor.execute(*query)
records = cursor.fetchall() records = cursor.fetchall()
for record in records: for record in records:
(book_id, values) = values_from_record(record) (book_id, values) = values_from_record(record)
result_cache[book_id] = values
result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref
# aggregate sub-cashbooks to requested cashbooks from cache for name in ['current_value', 'diff_amount',
for id_none in ids_nonebtypes: 'current_value_ref', 'purchase_amount_ref']:
records = CBook.search([ if result[name][book_id] is None:
('btype.feature', '=', 'asset'), result[name][book_id] = Decimal('0.0')
('parent', 'child_of', [id_none])
])
values = {x:Decimal('0.0') for x in aggr_names+['purchase_amount_ref']} value = Decimal('0.0')
for record in records: if name == 'current_value':
for name in aggr_names+['purchase_amount_ref']: value = Currency.compute(
values[name] += \ company_currency if company_currency is not None else record[4],
result_cache.get(record.id, {}).get(name, Decimal('0.0'))
# convert current-value-ref in company-currency to
# currency of current cashbook
cbook = CBook(id_none)
values['current_value'] = Currency.compute(
company_currency if company_currency is not None else cbook.currency,
values['current_value_ref'], values['current_value_ref'],
cbook.currency, record[10],
) )
elif name == 'diff_amount':
values['diff_amount'] = Currency.compute( value = Currency.compute(
company_currency if company_currency is not None else cbook.currency, company_currency if company_currency is not None else record[4],
values['current_value_ref'] - values['purchase_amount_ref'], values['current_value_ref'] - values['purchase_amount_ref'],
cbook.currency, record[10],
) )
elif name in ['current_value_ref', 'purchase_amount_ref']:
value = values[name]
result['digits'][book_id] = record[5]
result[name][book_id] += value
values['diff_percent'] = \ # diff_prcent
(Decimal('100.0') * values['current_value_ref'] / \ for id_book in ids_nonebtypes:
values['purchase_amount_ref'] - Decimal('100.0') c_val = result['current_value_ref'][id_book]
).quantize( p_amount = result['purchase_amount_ref'][id_book]
Decimal(str(1/10**cbook.currency_digits)) digits = result['digits'][id_book]
) if values['purchase_amount_ref'] != Decimal('0.0') else None
for name in values.keys(): if (p_amount == Decimal('0.0')) or (p_amount is None) or (c_val is None):
if name not in result.keys(): continue
result[name] = {}
result[name][id_none] = values[name] result['diff_percent'][id_book] = (
Decimal('100.0') * c_val / p_amount - Decimal('100.0')
).quantize(Decimal(str(1/10 ** digits)))
result['digits'][id_book] = None
logger.warning('## get_asset_quantity-PART2 %(time)s' % {
'time': datetime.now().isoformat(),
})
# store to cache # store to cache
MemCache.store_result(cashbooks, cache_keys, result) MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook)
logger.warning('## get_asset_quantity-END %(time)s' % {
'time': datetime.now().isoformat(),
})
return {x:result[x] for x in names} return {x:result[x] for x in names}
@fields.depends('id') @fields.depends('id')