book: new field 'purchase_amount', optimize form

speedup: indexes, caching

book.py

@@ -15,12 +15,13 @@ from sql import Literal
 from sql.functions import CurrentDate
 from sql.aggregate import Sum
 from sql.conditionals import Case, Coalesce
+from trytond.modules.cashbook.model import CACHEKEY_CURRENCY
 
 
 class Book(SymbolMixin, metaclass=PoolMeta):
     __name__ = 'cashbook.book'
 
-    asset = fields.Many2One(string='Asset',
+    asset = fields.Many2One(string='Asset', select=True,
         model_name='investment.asset', ondelete='RESTRICT',
         states={
             'required': Eval('feature', '') == 'asset',
@@ -47,7 +48,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
     asset_uomcat = fields.Function(fields.Many2One(string='UOM Category',
         readonly=True, model_name='product.uom.category',
         states={'invisible': True}), 'on_change_with_asset_uomcat')
-    quantity_uom = fields.Many2One(string='UOM',
+    quantity_uom = fields.Many2One(string='UOM', select=True,
         model_name='product.uom', ondelete='RESTRICT',
         domain=[
             ('category.id', '=', Eval('asset_uomcat', -1)),
@@ -118,6 +119,13 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         states={
             'invisible': Eval('feature', '') != 'asset',
         }, depends=['currency_digits', 'feature']), 'get_asset_quantity')
+    purchase_amount = fields.Function(fields.Numeric(string='Purchase Amount',
+        help='Total purchase amount, from shares and fees.',
+        readonly=True, digits=(16, Eval('currency_digits', 2)),
+        states={
+            'invisible': Eval('feature', '') != 'asset',
+        }, depends=['currency_digits', 'feature']),
+        'get_asset_quantity')
 
     # yield
     yield_dividend_total = fields.Function(fields.Numeric(string='Dividend',
@@ -200,14 +208,13 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         """
         return 4
 
-    @fields.depends('yield_sales', 'yield_fee_total', 'yield_dividend_total', 'diff_amount')
+    @fields.depends('yield_sales', 'yield_dividend_total', 'diff_amount')
     def on_change_with_yield_balance(self, name=None):
         """ calculate yield total
+            fee is already contained in 'diff_amount'
         """
         sum_lst = [self.diff_amount, self.yield_dividend_total, self.yield_sales]
         sum2 = sum([x for x in sum_lst if x is not None])
-        if self.yield_fee_total is not None:
-            sum2 -= self.yield_fee_total
         return sum2
 
     @classmethod
@@ -250,9 +257,14 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         pool = Pool()
         CashBook = pool.get('cashbook.book')
         IrDate = pool.get('ir.date')
+        MemCache = pool.get('cashbook.memcache')
         cursor = Transaction().connection.cursor()
         context = Transaction().context
-        result = {x:{y.id: Decimal('0.0') for y in cashbooks} for x in names}
+        result = {
+            x:{y.id: Decimal('0.0') for y in cashbooks}
+                for x in ['yield_fee_total', 'yield_dividend_total',
+                    'yield_sales', 'yield_fee_12m', 'yield_dividend_12m',
+                    'yield_sales_12m']}
 
         def quantize_val(value, line):
             """ quantize...
@@ -261,23 +273,50 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                     value or Decimal('0.0')
                 ).quantize(Decimal(str(1/10**line.currency_digits)))
 
+        query_date = context.get('date', IrDate.today())
+        cache_keys = {
+                x.id: MemCache.get_key_by_record(
+                    name = 'get_yield_data',
+                    record = x,
+                    query = [{
+                            'model': 'cashbook.line',
+                            'query': [('cashbook.parent', 'child_of', [x.id])],
+                        }, {
+                            'model': 'currency.currency.rate',
+                            'query': [('currency.id', '=', x.currency.id)],
+                            'cachekey': CACHEKEY_CURRENCY % x.currency.id,
+                        }, {
+                            'model': 'investment.rate',
+                            'query': [('asset.id', '=', x.asset.id)],
+                        } if x.asset is not None else {}],
+                    addkeys = [query_date.isoformat()])
+                for x in cashbooks
+            }
+
+        # read from cache
+        (todo_cashbook, result) = MemCache.read_from_cache(
+            cashbooks, cache_keys, names, result)
+        if len(todo_cashbook) == 0:
+            return result
+
         # results for 'total'
+        records_total = []
+        records_12m = []
+        if len(todo_cashbook) > 0:
             (tab_book1, query_total) = cls.get_yield_data_sql()
-        query_total.where &= tab_book1.id.in_([x.id for x in cashbooks])
+            query_total.where &= tab_book1.id.in_([x.id for x in todo_cashbook])
             cursor.execute(*query_total)
             records_total = cursor.fetchall()
 
             # results for 12 months
-        query_date = context.get('date', IrDate.today())
             (tab_book2, query_12m) = cls.get_yield_data_sql(
                     date_to = query_date,
                     date_from = query_date - timedelta(days=365),
                 )
-        query_12m.where &= tab_book2.id.in_([x.id for x in cashbooks])
+            query_12m.where &= tab_book2.id.in_([x.id for x in todo_cashbook])
             cursor.execute(*query_12m)
             records_12m = cursor.fetchall()
 
-        result = {x:{y.id: Decimal('0.0') for y in cashbooks} for x in names}
         for record in records_total:
             book = CashBook(record[0])
             result['yield_fee_total'][record[0]] = quantize_val(record[1], book)
@@ -290,7 +329,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
             result['yield_dividend_12m'][record[0]] = quantize_val(record[2], book)
             result['yield_sales_12m'][record[0]] = quantize_val(record[3], book)
 
-        return result
+        # store to cache
+        MemCache.store_result(cashbooks, cache_keys, result)
+        return {x:result[x] for x in names}
 
     @classmethod
     def get_asset_quantity_sql(cls):
@@ -309,6 +350,7 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         tab_asset = Asset.__table__()
         (tab_rate, tab2) = Asset.get_rate_data_sql()
         (tab_balance, tab2) = CBook.get_balance_of_cashbook_sql()
+        (tab_line_yield, query_yield) = Line.get_yield_data_sql()
         context = Transaction().context
 
         query_date = context.get('qdate', CurrentDate())
@@ -320,6 +362,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                 condition=tab_book.currency==tab_cur.id,
             ).join(tab_asset,
                 condition=tab_book.asset==tab_asset.id,
+            ).join(query_yield,
+                condition=query_yield.id==tab_line.id,
             ).join(tab_balance,
                 condition=tab_book.id==tab_balance.cashbook,
                 type_ = 'LEFT OUTER',
@@ -341,6 +385,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                 tab_book.quantity_uom,  # 7
                 tab_asset.currency.as_('asset_currency'), #8
                 Coalesce(tab_balance.balance, Decimal('0.0')).as_('balance'), #9
+                (
+                    Sum(query_yield.fee) + tab_balance.balance
+                ).as_('purchase_amount'), #10
                 group_by=[tab_book.id, tab_rate.rate,
                     tab_book.currency, tab_cur.digits, tab_asset.uom,
                     tab_book.quantity_uom, tab_asset.currency,
@@ -357,6 +404,8 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         CBook = pool.get('cashbook.book')
         Uom = pool.get('product.uom')
         Currency = pool.get('currency.currency')
+        IrDate = pool.get('ir.date')
+        MemCache = pool.get('cashbook.memcache')
         cursor = Transaction().connection.cursor()
         context = Transaction().context
         (query, tab_book) = cls.get_asset_quantity_sql()
@@ -364,6 +413,33 @@ class Book(SymbolMixin, metaclass=PoolMeta):
         company_currency = CBook.default_currency()
         result = {x:{y.id: None for y in cashbooks} for x in names}
 
+        cache_keys = {
+                x.id: MemCache.get_key_by_record(
+                    name = 'get_asset_quantity',
+                    record = x,
+                    query = [{
+                            'model': 'cashbook.line',
+                            'query': [('cashbook.parent', 'child_of', [x.id])],
+                        }, {
+                            'model': 'currency.currency.rate',
+                            'query': [('currency.id', '=', x.currency.id)],
+                        }, {
+                            'model': 'investment.rate',
+                            'query': [('asset.id', '=', x.asset.id)],
+                        } if x.asset is not None else {}],
+                    addkeys=[
+                            str(company_currency),
+                            str(context.get('qdate', IrDate.today()).toordinal()),
+                        ])
+                for x in cashbooks
+            }
+
+        # read from cache
+        (todo_cashbook, result) = MemCache.read_from_cache(
+            cashbooks, cache_keys, names, result)
+        if len(todo_cashbook) == 0:
+            return result
+
         def values_from_record(rdata):
             """ compute values for record
             """
@@ -389,16 +465,22 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                         rdata[3] * rdata[1] / uom_factor,
                         company_currency if company_currency is not None else rdata[8],
                     ),
-                'diff_amount': current_value - rdata[9],
+                'diff_amount': current_value - rdata[10],
                 'diff_percent': (
                         Decimal('100.0') * current_value / \
-                        rdata[9] - Decimal('100.0')
+                        rdata[10] - Decimal('100.0')
                     ).quantize(Decimal(str(1/10**rdata[5]))) \
-                        if rdata[9] != Decimal('0.0') else None,
+                        if rdata[10] != Decimal('0.0') else None,
                 'current_rate': (
                         current_value / rdata[1]
                     ).quantize(Decimal(str(1/10**rdata[5]))) \
                         if rdata[1] != Decimal('0.0') else None,
+                'purchase_amount': record[10].quantize(Decimal(str(1/10**rdata[5]))),
+                'purchase_amount_ref': Currency.compute(
+                        rdata[4],
+                        record[10],
+                        company_currency if company_currency is not None else rdata[4],
+                    ),
                 })
 
         result_cache = {}
@@ -416,7 +498,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                 result_cache[book_id] = values
                 result_cache[book_id]['balance_ref'] = CBook(book_id).balance_ref
 
-                for name in names:
+                for name in values.keys():
+                    if name not in result.keys():
+                        result[name] = {}
                     result[name][book_id] = values[name]
 
         # add aggregated values of cashbooks without type
@@ -451,9 +535,9 @@ class Book(SymbolMixin, metaclass=PoolMeta):
                         ('parent', 'child_of', [id_none])
                     ])
 
-                values = {x:Decimal('0.0') for x in aggr_names+['balance_ref']}
+                values = {x:Decimal('0.0') for x in aggr_names+['purchase_amount_ref']}
                 for record in records:
-                    for name in aggr_names+['balance_ref']:
+                    for name in aggr_names+['purchase_amount_ref']:
                         values[name] += \
                             result_cache.get(record.id, {}).get(name, Decimal('0.0'))
 
@@ -468,21 +552,24 @@ class Book(SymbolMixin, metaclass=PoolMeta):
 
                 values['diff_amount'] = Currency.compute(
                         company_currency if company_currency is not None else cbook.currency,
-                        values['current_value_ref'] - values['balance_ref'],
+                        values['current_value_ref'] - values['purchase_amount_ref'],
                         cbook.currency,
                     )
 
                 values['diff_percent'] = \
                     (Decimal('100.0') * values['current_value_ref'] / \
-                        values['balance_ref'] - Decimal('100.0')
+                        values['purchase_amount_ref'] - Decimal('100.0')
                     ).quantize(
                             Decimal(str(1/10**cbook.currency_digits))
-                        ) if values['balance_ref'] != Decimal('0.0') else None
+                        ) if values['purchase_amount_ref'] != Decimal('0.0') else None
-
+                for name in values.keys():
-                for name in queried_names:
+                    if name not in result.keys():
+                        result[name] = {}
                     result[name][id_none] = values[name]
 
-        return result
+        # store to cache
+        MemCache.store_result(cashbooks, cache_keys, result)
+        return {x:result[x] for x in names}
 
     @fields.depends('id')
     def on_change_with_show_performance(self, name=None):

locale/de.po

@@ -198,10 +198,13 @@ msgctxt "help:cashbook.book,yield_balance:"
 msgid "Total income: price gain + dividends + sales gains - fees"
 msgstr "Gesamtertrag: Kursgewinn + Dividenden + Verkaufsgewinne - Gebühren"
 
+msgctxt "field:cashbook.book,purchase_amount:"
+msgid "Purchase Amount"
+msgstr "Kaufbetrag"
 
-# Total return - share price gain plus dividend minus fees
+msgctxt "help:cashbook.book,purchase_amount:"
-# Total return - price gain + sales gain + dividend - fees
+msgid "Total purchase amount, from shares and fees."
-# Rendite insgesamt - Kursgewinn + Verkaufserfolg + Dividende - Gebühren
+msgstr "Kaufbetrag gesamt, aus Anteilen und Gebühren."
 
 
 ##################

locale/en.po

@@ -182,6 +182,14 @@ msgctxt "help:cashbook.book,yield_balance:"
 msgid "Total income: price gain + dividends + sales gains - fees"
 msgstr "Total income: price gain + dividends + sales gains - fees"
 
+msgctxt "field:cashbook.book,purchase_amount:"
+msgid "Purchase Amount"
+msgstr "Purchase Amount"
+
+msgctxt "help:cashbook.book,purchase_amount:"
+msgid "Total purchase amount, from shares and fees."
+msgstr "Total purchase amount, from shares and fees."
+
 msgctxt "field:cashbook.split,quantity_digits:"
 msgid "Digits"
 msgstr "Digits"

tests/test_book.py

@@ -432,7 +432,7 @@ class CbInvTestCase(CashbookTestCase, InvestmentTestCase):
             self.assertEqual(book2.quantity_all, Decimal('20.0'))
             # usd --> eur: 1750 US$ / 1.05 = 1666.666 €
             # 1 ounce --> 20 gram: 1666.666 € * 20 / 28.3495 = 1175.7996 €
-            # bette we use 'Troy Ounce': 1 oz.tr. = 31.1034768 gram
+            # better we use 'Troy Ounce': 1 oz.tr. = 31.1034768 gram
             self.assertEqual(book2.current_value, Decimal('1175.80'))
             self.assertEqual(book2.current_value_ref, Decimal('1175.80'))
             self.assertEqual(book2.diff_amount, Decimal('-74.20'))

tests/test_yield.py

@@ -160,7 +160,7 @@ class YieldTestCase(ModuleTestCase):
         self.assertEqual(book_asset.yield_dividend_total, Decimal('23.5'))
         self.assertEqual(book_asset.yield_fee_total, Decimal('4.0'))
         self.assertEqual(book_asset.yield_sales, Decimal('0.0'))
-        self.assertEqual(book_asset.diff_amount, Decimal('-19.5'))
+        self.assertEqual(book_asset.diff_amount, Decimal('-23.5'))
         self.assertEqual(book_asset.yield_balance, Decimal('0.0'))
 
     @with_transaction()

view/book_form.xml

@@ -8,25 +8,29 @@ full copyright notices and license terms. -->
         <separator colspan="4" name="current_value" string="Current valuation of the investment"/>
     </xpath>
     <xpath expr="/form/notebook/page[@name='balance']/field[@name='balance']" position="after">
-        <label name="current_value"/>
+        <label name="purchase_amount"/>
-        <field name="current_value" symbol="currency"/>
+        <field name="purchase_amount" symbol="currency"/>
-        <label name="current_value_ref"/>
+        <label name="yield_balance"/>
-        <field name="current_value_ref" symbol="company_currency"/>
+        <field name="yield_balance" symbol="currency"/>
     </xpath>
     <xpath expr="/form/notebook/page[@name='balance']/field[@name='balance_all']" position="after">
+        <label name="current_value"/>
+        <field name="current_value" symbol="currency"/>
         <label name="diff_amount"/>
         <field name="diff_amount" symbol="currency"/>
+    </xpath>
+    <xpath expr="/form/notebook/page[@name='balance']/field[@name='balance_ref']" position="after">
+        <label name="current_rate"/>
+        <field name="current_rate" symbol="asset_symbol"/>
         <label name="diff_percent"/>
         <group id="diff_percent" col="2">
             <field name="diff_percent" xexpand="0"/>
             <label name="diff_percent" xalign="0.0" string="%" xexpand="1"/>
         </group>
-    </xpath>
+
-    <xpath expr="/form/notebook/page[@name='balance']/field[@name='balance_ref']" position="after">
+        <label name="current_value_ref" colspan="2" string=" "/>
-        <label name="yield_balance"/>
+        <label name="current_value_ref"/>
-        <field name="yield_balance" symbol="currency"/>
+        <field name="current_value_ref" symbol="company_currency"/>
-        <label name="current_rate"/>
-        <field name="current_rate" symbol="asset_symbol"/>
         <newline/>
 
         <separator colspan="2" name="quantity" string="Quantity"/>
@@ -52,7 +56,6 @@ full copyright notices and license terms. -->
         <field name="yield_dividend_12m" symbol="currency"/>
         <label name="yield_fee_12m"/>
         <field name="yield_fee_12m" symbol="currency"/>
-
     </xpath>
 
     <xpath expr="/form/notebook/page[@id='pggeneral']" position="after">