mdsfred/cashbook_investment
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

book: optimize query for speed

Browse source
This commit is contained in:
Frederik Jaeckel 2023-03-04 21:21:54 +01:00
parent e6bb17f517
commit 6c5be8df2d
1 changed files with 158 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

256
book.py
View file

@ -10,14 +10,18 @@ from trytond.modules.cashbook.book import STATES2, DEPENDS2
from trytond.transaction import Transaction
from trytond.report import Report
from decimal import Decimal
from datetime import timedelta
from datetime import timedelta, datetime
from sql import Literal
from sql.functions import CurrentDate
from sql.aggregate import Sum
from sql.conditionals import Case, Coalesce
from trytond.modules.cashbook.model import CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK
from trytond.modules.cashbook.model import sub_ids_hierarchical,\
CACHEKEY_CURRENCY, CACHEKEY_CASHBOOK, AnyInArray
from .asset import CACHEKEY_ASSETRATE
import logging
logger = logging.getLogger(__name__)
class Book(SymbolMixin, metaclass=PoolMeta):
__name__ = 'cashbook.book'
@ -224,10 +228,12 @@ class Book(SymbolMixin, metaclass=PoolMeta):
"""
pool = Pool()
Line = pool.get('cashbook.line')
Currency = pool.get('currency.currency')
(tab_line1, tab_line_yield) = Line.get_yield_data_sql()
(tab_line2, tab_line_gainloss) = Line.get_gainloss_data_sql()
tab_book = cls.__table__()
tab_line = Line.__table__()
tab_cur = Currency.__table__()
where = Literal(True)
if date_from:
@ -237,6 +243,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
query = tab_book.join(tab_line,
condition=tab_line.cashbook==tab_book.id,
).join(tab_cur,
condition=tab_cur.id==tab_book.currency,
).join(tab_line_yield,
condition=tab_line_yield.id==tab_line.id,
).join(tab_line_gainloss,
@ -246,7 +254,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
Sum(tab_line_yield.fee).as_('fee'),
Sum(tab_line_yield.dividend).as_('dividend'),
Sum(tab_line_gainloss.gainloss).as_('gainloss'),
group_by=[tab_book.id],
tab_cur.digits.as_('currency_digits'),
group_by=[tab_book.id, tab_cur.digits],
where=where
)
return (tab_book, query)
@ -267,12 +276,18 @@ class Book(SymbolMixin, metaclass=PoolMeta):
'yield_sales', 'yield_fee_12m', 'yield_dividend_12m',
'yield_sales_12m']}
def quantize_val(value, line):
logger.warning('## get_yield_data-GO %(time)s %(ids)s %(name)s' % {
'time': datetime.now().isoformat(),
'ids': str([x.id for x in cashbooks]),
'name': names,
})
def quantize_val(value, digits):
""" quantize...
"""
return (
value or Decimal('0.0')
).quantize(Decimal(str(1/10**line.currency_digits)))
).quantize(Decimal(str(1/10 ** digits)))
query_date = context.get('date', IrDate.today())
cache_keys = {
@ -282,7 +297,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
query = [{
'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])],
'cachekey': CACHEKEY_CASHBOOK % x.id,
#'cachekey': CACHEKEY_CASHBOOK % x.id,
}, {
'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)],
@ -290,16 +305,22 @@ class Book(SymbolMixin, metaclass=PoolMeta):
}, {
'model': 'investment.rate',
'query': [('asset.id', '=', x.asset.id)],
'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
#'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
} if x.asset is not None else {}],
addkeys = [query_date.isoformat()])
for x in cashbooks
}
logger.warning('## get_yield_data-KEYS %(time)s' % {
'time': datetime.now().isoformat(),
})
# read from cache
(todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0:
logger.warning('## get_yield_data-HIT %(time)s' % {
'time': datetime.now().isoformat(),
})
return result
# results for 'total'
@ -321,19 +342,20 @@ class Book(SymbolMixin, metaclass=PoolMeta):
records_12m = cursor.fetchall()
for record in records_total:
book = CashBook(record[0])
result['yield_fee_total'][record[0]] = quantize_val(record[1], book)
result['yield_dividend_total'][record[0]] = quantize_val(record[2], book)
result['yield_sales'][record[0]] = quantize_val(record[3], book)
result['yield_fee_total'][record[0]] = quantize_val(record[1], record[4])
result['yield_dividend_total'][record[0]] = quantize_val(record[2], record[4])
result['yield_sales'][record[0]] = quantize_val(record[3], record[4])
for record in records_12m:
book = CashBook(record[0])
result['yield_fee_12m'][record[0]] = quantize_val(record[1], book)
result['yield_dividend_12m'][record[0]] = quantize_val(record[2], book)
result['yield_sales_12m'][record[0]] = quantize_val(record[3], book)
result['yield_fee_12m'][record[0]] = quantize_val(record[1], record[4])
result['yield_dividend_12m'][record[0]] = quantize_val(record[2], record[4])
result['yield_sales_12m'][record[0]] = quantize_val(record[3], record[4])
# store to cache
MemCache.store_result(cashbooks, cache_keys, result)
MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook)
logger.warning('## get_yield_data-END %(time)s' % {
'time': datetime.now().isoformat(),
})
return {x:result[x] for x in names}
@classmethod
@ -387,10 +409,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
tab_asset.uom, # 6
tab_book.quantity_uom, # 7
tab_asset.currency.as_('asset_currency'), #8
Coalesce(tab_balance.balance, Decimal('0.0')).as_('balance'), #9
(
Sum(query_yield.fee) + tab_balance.balance
).as_('purchase_amount'), #10
).as_('purchase_amount'), #9
group_by=[tab_book.id, tab_rate.rate,
tab_book.currency, tab_cur.digits, tab_asset.uom,
tab_book.quantity_uom, tab_asset.currency,
@ -402,6 +423,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
@classmethod
def get_asset_quantity(cls, cashbooks, names):
""" get quantities
field: quantity, quantity_all, current_value,
current_value_ref, diff_amount, diff_percent,
current_rate, purchase_amount
"""
pool = Pool()
CBook = pool.get('cashbook.book')
@ -414,36 +438,54 @@ class Book(SymbolMixin, metaclass=PoolMeta):
(query, tab_book) = cls.get_asset_quantity_sql()
company_currency = CBook.default_currency()
result = {x:{y.id: None for y in cashbooks} for x in names}
result = {
x:{y.id: None for y in cashbooks}
for x in ['quantity', 'quantity_all', 'current_value',
'current_value_ref', 'diff_amount', 'diff_percent',
'current_rate', 'purchase_amount', 'purchase_amount_ref',
'digits']
}
logger.warning('## get_asset_quantity-GO %(time)s %(ids)s %(name)s' % {
'time': datetime.now().isoformat(),
'ids': str([x.id for x in cashbooks]),
'name': names,
})
cache_keys = {
x.id: MemCache.get_key_by_record(
name = 'get_asset_quantity',
record = x,
query = [{
'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])],
'cachekey': CACHEKEY_CASHBOOK % x.id,
}, {
'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)],
'cachekey': CACHEKEY_CURRENCY % x.currency.id,
}, {
'model': 'investment.rate',
'query': [('asset.id', '=', x.asset.id)],
'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
} if x.asset is not None else {}],
addkeys=[
str(company_currency),
str(context.get('qdate', IrDate.today()).toordinal()),
])
for x in cashbooks
x.id: MemCache.get_key_by_record(
name = 'get_asset_quantity',
record = x,
query = [{
'model': 'cashbook.line',
'query': [('cashbook.parent', 'child_of', [x.id])],
#'cachekey': CACHEKEY_CASHBOOK % x.id,
}, {
'model': 'currency.currency.rate',
'query': [('currency.id', '=', x.currency.id)],
'cachekey': CACHEKEY_CURRENCY % x.currency.id,
}, {
'model': 'investment.rate',
'query': [('asset.id', '=', x.asset.id)],
#'cachekey': CACHEKEY_ASSETRATE % x.asset.id,
} if x.asset is not None else {}],
addkeys=[
str(company_currency),
str(context.get('qdate', IrDate.today()).toordinal()),
])
for x in cashbooks
}
# read from cache
logger.warning('## get_asset_quantity-KEYS %(time)s' % {
'time': datetime.now().isoformat(),
})
(todo_cashbook, result) = MemCache.read_from_cache(
cashbooks, cache_keys, names, result)
if len(todo_cashbook) == 0:
logger.warning('## get_asset_quantity-HIT %(time)s' % {
'time': datetime.now().isoformat(),
})
return result
def values_from_record(rdata):
@ -471,25 +513,24 @@ class Book(SymbolMixin, metaclass=PoolMeta):
rdata[3] * rdata[1] / uom_factor,
company_currency if company_currency is not None else rdata[8],
),
'diff_amount': current_value - rdata[10],
'diff_amount': current_value - rdata[9],
'diff_percent': (
Decimal('100.0') * current_value / \
rdata[10] - Decimal('100.0')
rdata[9] - Decimal('100.0')
).quantize(Decimal(str(1/10**rdata[5]))) \
if rdata[10] != Decimal('0.0') else None,
if rdata[9] != Decimal('0.0') else None,
'current_rate': (
current_value / rdata[1]
).quantize(Decimal(str(1/10**rdata[5]))) \
if rdata[1] != Decimal('0.0') else None,
'purchase_amount': record[10].quantize(Decimal(str(1/10**rdata[5]))),
'purchase_amount': record[9].quantize(Decimal(str(1/10**rdata[5]))),
'purchase_amount_ref': Currency.compute(
rdata[4],
record[10],
record[9],
company_currency if company_currency is not None else rdata[4],
),
})
result_cache = {}
ids_assetbooks = [x.id for x in cashbooks if x.btype is not None]
ids_nonebtypes = [x.id for x in cashbooks if x.btype is None]
@ -501,80 +542,99 @@ class Book(SymbolMixin, metaclass=PoolMeta):
for record in records:
(book_id, values) = values_from_record(record)
result_cache[book_id] = values
result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref
for name in values.keys():
if name not in result.keys():
result[name] = {}
result[name][book_id] = values[name]
logger.warning('## get_asset_quantity-PART1 %(time)s' % {
'time': datetime.now().isoformat(),
})
# add aggregated values of cashbooks without type
aggr_names = ['current_value', 'current_value_ref',
'diff_amount', 'diff_percent']
queried_names = list(set(aggr_names).intersection(set(names)))
if len(queried_names) > 0:
# query all subordered asset-cashbooks for
# btype=None-cashbooks
query1 = [('btype.feature', '=', 'asset'),
('parent', 'child_of', ids_nonebtypes)]
if len(result_cache.keys()) > 0:
query1.append(('id', 'not in', result_cache.keys()))
books_query = CBook.search(query1, query=True)
if (len(queried_names) > 0) and (len(ids_nonebtypes) > 0):
# query all subordered asset-cashbooks to get values for
# cashbooks without type
# add results to cache
(query, tab_book) = cls.get_asset_quantity_sql()
query.where &= tab_book.id.in_(books_query)
(tab_quantity, tab_book) = cls.get_asset_quantity_sql()
tab_subids = sub_ids_hierarchical('cashbook.book')
query = tab_book.join(tab_subids,
condition=tab_book.id==tab_subids.parent,
).join(tab_quantity,
condition=tab_quantity.id==AnyInArray(tab_subids.subids),
).select(
tab_book.id,
Sum(tab_quantity.quantity), # 1
Sum(tab_quantity.quantity_all), # 2
tab_quantity.rate, # 3
tab_quantity.currency, # 4
tab_quantity.currency_digits, # 5
tab_quantity.uom, # 6
tab_quantity.quantity_uom, # 7
tab_quantity.asset_currency, # 8
Sum(tab_quantity.purchase_amount), # 9
tab_book.currency.as_('currency_book'), # 10
where=tab_book.id.in_(ids_nonebtypes),
group_by=[tab_book.id, tab_quantity.rate,
tab_quantity.currency, tab_quantity.currency_digits,
tab_quantity.uom, tab_quantity.quantity_uom,
tab_quantity.asset_currency,tab_book.currency],
)
cursor.execute(*query)
records = cursor.fetchall()
for record in records:
(book_id, values) = values_from_record(record)
result_cache[book_id] = values
result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref
# aggregate sub-cashbooks to requested cashbooks from cache
for id_none in ids_nonebtypes:
records = CBook.search([
('btype.feature', '=', 'asset'),
('parent', 'child_of', [id_none])
])
for name in ['current_value', 'diff_amount',
'current_value_ref', 'purchase_amount_ref']:
if result[name][book_id] is None:
result[name][book_id] = Decimal('0.0')
values = {x:Decimal('0.0') for x in aggr_names+['purchase_amount_ref']}
for record in records:
for name in aggr_names+['purchase_amount_ref']:
values[name] += \
result_cache.get(record.id, {}).get(name, Decimal('0.0'))
value = Decimal('0.0')
if name == 'current_value':
value = Currency.compute(
company_currency if company_currency is not None else record[4],
values['current_value_ref'],
record[10],
)
elif name == 'diff_amount':
value = Currency.compute(
company_currency if company_currency is not None else record[4],
values['current_value_ref'] - values['purchase_amount_ref'],
record[10],
)
elif name in ['current_value_ref', 'purchase_amount_ref']:
value = values[name]
result['digits'][book_id] = record[5]
result[name][book_id] += value
# convert current-value-ref in company-currency to
# currency of current cashbook
cbook = CBook(id_none)
values['current_value'] = Currency.compute(
company_currency if company_currency is not None else cbook.currency,
values['current_value_ref'],
cbook.currency,
)
# diff_prcent
for id_book in ids_nonebtypes:
c_val = result['current_value_ref'][id_book]
p_amount = result['purchase_amount_ref'][id_book]
digits = result['digits'][id_book]
values['diff_amount'] = Currency.compute(
company_currency if company_currency is not None else cbook.currency,
values['current_value_ref'] - values['purchase_amount_ref'],
cbook.currency,
)
if (p_amount == Decimal('0.0')) or (p_amount is None) or (c_val is None):
continue
values['diff_percent'] = \
(Decimal('100.0') * values['current_value_ref'] / \
values['purchase_amount_ref'] - Decimal('100.0')
).quantize(
Decimal(str(1/10**cbook.currency_digits))
) if values['purchase_amount_ref'] != Decimal('0.0') else None
for name in values.keys():
if name not in result.keys():
result[name] = {}
result[name][id_none] = values[name]
result['diff_percent'][id_book] = (
Decimal('100.0') * c_val / p_amount - Decimal('100.0')
).quantize(Decimal(str(1/10 ** digits)))
result['digits'][id_book] = None
logger.warning('## get_asset_quantity-PART2 %(time)s' % {
'time': datetime.now().isoformat(),
})
# store to cache
MemCache.store_result(cashbooks, cache_keys, result)
MemCache.store_result(cashbooks, cache_keys, result, todo_cashbook)
logger.warning('## get_asset_quantity-END %(time)s' % {
'time': datetime.now().isoformat(),
})
return {x:result[x] for x in names}
@fields.depends('id')