optimize code

pre defned online sources: add FAZ.net

README.rst

@@ -9,7 +9,7 @@ pip install mds-investment
 
 Requires
 ========
-- Tryton 6.0
+- Tryton 7.0
 
 How to
 ======
@@ -22,110 +22,6 @@ You can define the course sources yourself.
 Changes
 =======
 
-*6.0.24 - 07.06.2023*
+*7.0.0 - 01.12.2023*
 
-- code optimized
-
-*6.0.23 - 21.01.2023*
-
-- updt: online-source optimized for external extension
-
-*6.0.22 - 10.01.2023*
-
-- updt: optimze rate, remove caching
-
-*6.0.21 - 08.01.2023*
-
-- add: caching for percentual changes
-
-*6.0.20 - 07.01.2023*
-
-- updt: asset - percent-values optimize for speed
-
-*6.0.19 - 05.01.2023*
-
-- updt: rate - optimize for speed
-
-*6.0.18 - 04.01.2023*
-
-- add: units to lists
-- updt: online-sources
-
-*6.0.17 - 19.12.2022*
-
-- add: import wizard
-
-*6.0.16 - 18.12.2022*
-
-- add: onlinesource - sorting, sources
-
-*6.0.15 - 16.12.2022*
-
-- fix: recoursion
-- add: enable after-update actions
-
-*6.0.14 - 09.12.2022*
-
-- updt: translated symbol of asset-currency
-
-*6.0.13 - 06.12.2022*
-
-- updt: asset-form - units for percent-values, asset-list - day-negative in red-color
-
-*6.0.12 - 05.12.2022*
-
-- import-script - add field-delimter to param-list
-
-*6.0.11 - 03.12.2022*
-
-- add: tabs for asset-list, new icons, queries optimized
-
-*6.0.10 - 01.12.2022*
-
-- updt: optimized asset-list, unit as symbol
-
-*6.0.9 - 30.11.2022*
-
-- add: script to import historical rates
-
-*6.0.8 - 29.11.2022*
-
-- add: online-source as list to allow multiple sources
-       for asset
-
-*6.0.7 - 28.11.2022*
-
-- fix: add diagram-interpolate
-- fix: corrected nextupdate
-
-*6.0.6 - 26.11.2022*
-
-- add: diagram-display
-
-*6.0.5 - 25.11.2022*
-
-- add: sorter for name/nsin/isin/symbol
-- add: columns percentage by day/month/3month/6month/12month
-- fix: online-updater
-
-*6.0.4 - 25.11.2022*
-
-- add: assets - colors for percentual success/loss
-- updt: optimize timestamp for next online-update
-
-*6.0.3 - 23.11.2022*
-
-- fix: bug in searcher
-- add: online-sources
-
-*6.0.2 - 23.11.2022*
-
-- asset: add field 'date', optimized 'rec_name'
-
-*6.0.1 - 22.11.2022*
-
-- works
-
-*6.0.0 - 09.11.2022*
-
-- init
+- compatibility to Tryton 7.0

asset.py

@@ -3,7 +3,7 @@
 # The COPYRIGHT file at the top level of this repository contains the
 # full copyright notices and license terms.
 
-from trytond.model import ModelView, ModelSQL, fields, SymbolMixin
+from trytond.model import ModelView, ModelSQL, fields, SymbolMixin, Index
 from trytond.transaction import Transaction
 from trytond.pool import Pool
 from trytond.pyson import Eval, Bool, If, Date
@@ -15,6 +15,7 @@ from sql.functions import CurrentDate, CurrentTimestamp, Round, Extract
 from sql.conditionals import Case, Coalesce, NullIf
 from sql import Literal
 from .diagram import Concat2
+from .const import DEF_NONE
 
 
 digits_percent = 2
@@ -31,7 +32,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
     name = fields.Function(fields.Char(
         string='Name', readonly=True),
-            'get_name_symbol', searcher='search_rec_name')
+        'get_name_symbol', searcher='search_rec_name')
     company = fields.Many2One(
         string='Company', model_name='company.company',
         required=True, ondelete="RESTRICT")
@@ -45,12 +46,9 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
     uom = fields.Many2One(
         string='UOM', required=True, model_name='product.uom',
         ondelete='RESTRICT',
-        states={
-            'readonly': ~Bool(Eval('product')),
-        },
-        domain=[
-            ('category', '=', Eval('product_uom')),
-        ], depends=['product_uom', 'product'])
+        states={'readonly': ~Bool(Eval('product'))},
+        domain=[('category', '=', Eval('product_uom'))],
+        depends=['product_uom', 'product'])
     symbol = fields.Function(fields.Char(
         string='UOM', readonly=True), 'get_name_symbol',
         searcher='search_uom_symbol')
@@ -69,13 +67,11 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         'get_rate_data', searcher='search_date')
 
     currency = fields.Many2One(
-        string='Currency', select=True, required=True,
+        string='Currency', required=True,
         model_name='currency.currency', ondelete='RESTRICT')
     currency_digits = fields.Integer(
         string='Digits', required=True,
-        domain=[
-            ('currency_digits', '>=', 0),
-            ('currency_digits', '<=', 6)])
+        domain=[('currency_digits', '>=', 0), ('currency_digits', '<=', 6)])
 
     wkn = fields.Function(fields.Char(
         string='NSIN', readonly=True,
@@ -100,8 +96,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         string='URL',
         help='URL for data retrieval.',
         states={
-            'invisible': Eval('updturl_enable', False) == False,
-            'required': Eval('updturl_enable', False) == True,
+            'invisible': ~Eval('updturl_enable', False),
+            'required': Eval('updturl_enable', False),
         }, depends=['updturl_enable'])
     updturl_enable = fields.Function(fields.Boolean(
         string='URL required', readonly=True,
@@ -111,9 +107,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         string='Select days', required=True, selection=sel_updtdays)
     updttime = fields.Time(
         string='Time',
-        states={
-            'readonly': ~Bool(Eval('updtsources')),
-        }, depends=['updtsources'])
+        states={'readonly': ~Bool(Eval('updtsources'))},
+        depends=['updtsources'])
     nextupdate = fields.Function(fields.DateTime(
         string='Next Update', readonly=True),
         'get_nextupdates', searcher='search_nextupdate')
@@ -159,6 +154,21 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         super(Asset, cls).__setup__()
         cls._order.insert(0, ('name', 'ASC'))
         cls._order.insert(0, ('date', 'DESC'))
+        t = cls.__table__()
+        cls._sql_indexes.update({
+            Index(
+                t,
+                (t.product, Index.Equality())),
+            Index(
+                t,
+                (t.currency, Index.Equality())),
+            Index(
+                t,
+                (t.uom, Index.Equality())),
+            Index(
+                t,
+                (t.updtdays, Index.Equality())),
+            })
 
     @classmethod
     def migrate_updtsource(cls, module_name):
@@ -176,7 +186,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
             query = tab_asset.select(
                     tab_asset.id,
                     tab_asset.updtsource,
-                    where=tab_asset.updtsource != None,
+                    where=tab_asset.updtsource != DEF_NONE,
                 )
             cursor.execute(*query)
             records = cursor.fetchall()
@@ -185,7 +195,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                     'source': x[1],
                 } for x in records]
 
-            if len(to_create) > 0:
+            if to_create:
                 AssetSourceRel.create(to_create)
             asset_table.drop_column('updtsource')
 
@@ -246,7 +256,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
     def on_change_updtsources(self):
         """ clear time-fields
         """
-        if len(self.updtsources) == 0:
+        if not self.updtsources:
             self.updttime = None
         else:
             self.updttime = time(11, 30)
@@ -315,23 +325,24 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         """
         cursor = Transaction().connection.cursor()
 
-        (query, tab_asset) = cls.get_name_symbol_sql()
-        query.where = tab_asset.id.in_([x.id for x in assets])
-
-        cursor.execute(*query)
-        records = cursor.fetchall()
         result = {x: {y.id: None for y in assets} for x in names}
 
-        for record in records:
-            values = {
-                'name': record[1],
-                'product_uom': record[2],
-                'symbol': record[3],
-                'asset_symbol': record[0],
-                }
+        (query, tab_asset) = cls.get_name_symbol_sql()
+        if assets:
+            query.where = tab_asset.id.in_([x.id for x in assets])
+            cursor.execute(*query)
+            records = cursor.fetchall()
 
-            for name in names:
-                result[name][record[0]] = values[name]
+            for record in records:
+                values = {
+                    'name': record[1],
+                    'product_uom': record[2],
+                    'symbol': record[3],
+                    'asset_symbol': record[0],
+                    }
+
+                for name in names:
+                    result[name][record[0]] = values[name]
         return result
 
     @classmethod
@@ -369,32 +380,32 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
     def get_rate_data(cls, assets, names):
         """ get date and rate of asset
         """
-        Asset2 = Pool().get('investment.asset')
-
         cursor = Transaction().connection.cursor()
-        (query, tab_asset) = cls.get_rate_data_sql()
-        query.where = tab_asset.id.in_([x.id for x in assets])
-
-        cursor.execute(*query)
-        records = cursor.fetchall()
 
         result = {x: {y.id: None for y in assets} for x in names}
 
-        for record in records:
-            (id1, rate1, date1, id_rate) = record
+        if assets:
+            (query, tab_asset) = cls.get_rate_data_sql()
+            query.where = tab_asset.id.in_([x.id for x in assets])
+            curr_digits = {x.id: x.currency_digits for x in assets}
 
-            asset = Asset2(id1)
-            exp = Decimal(Decimal(1) / 10 ** (asset.currency_digits or 4))
+            cursor.execute(*query)
+            records = cursor.fetchall()
 
-            values = {
-                'rate': record[1].quantize(exp),
-                'date': record[2],
-                'change_symbol': id_rate,
-                }
+            for record in records:
+                (id1, rate1, date1, id_rate) = record
 
-            for name in names:
-                result[name][record[0]] = values[name]
+                curr_dig = curr_digits.get(id1, 4)
+                exp = Decimal(Decimal(1) / 10 ** curr_dig)
 
+                values = {
+                    'rate': record[1].quantize(exp),
+                    'date': record[2],
+                    'change_symbol': id_rate,
+                    }
+
+                for name in names:
+                    result[name][record[0]] = values[name]
         return result
 
     @classmethod
@@ -406,8 +417,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.id,
-                where=Operator(tab_query.date, clause[2]),
-            )
+                where=Operator(tab_query.date, clause[2]))
         return [('id', 'in', query)]
 
     @classmethod
@@ -419,8 +429,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.id,
-                where=Operator(tab_query.rate, clause[2]),
-            )
+                where=Operator(tab_query.rate, clause[2]))
         return [('id', 'in', query)]
 
     @staticmethod
@@ -432,8 +441,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.date,
-                where=tab_query.id == table.id,
-            )
+                where=tab_query.id == table.id)
         return [query]
 
     @staticmethod
@@ -445,8 +453,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.rate,
-                where=tab_query.id == table.id,
-            )
+                where=tab_query.id == table.id)
         return [query]
 
     @classmethod
@@ -457,8 +464,6 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         """
         pool = Pool()
         Rate = pool.get('investment.rate')
-        Asset2 = pool.get('investment.asset')
-        tab_asset = Asset2.__table__()
         tab_rate1 = Rate.__table__()
         tab_rate2 = Rate.__table__()
         context = Transaction().context
@@ -466,19 +471,15 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         query_date = context.get('qdate', CurrentDate())
         where_asset = tab_rate1.date <= query_date
         if isinstance(asset_ids, list):
-            where_asset &= tab_asset.id.in_(asset_ids)
+            where_asset &= tab_rate1.asset.in_(asset_ids)
 
-        tab_today = tab_asset.join(
-                tab_rate1,
-                condition=tab_asset.id == tab_rate1.asset,
-            ).select(
-                tab_asset.id,
+        tab_today = tab_rate1.select(
+                tab_rate1.asset.as_('id'),
                 tab_rate1.date,
                 tab_rate1.rate,
-                distinct_on=[tab_asset.id],
-                order_by=[tab_asset.id, tab_rate1.date.desc],
-                where=where_asset,
-            )
+                distinct_on=[tab_rate1.asset],
+                order_by=[tab_rate1.asset, tab_rate1.date.desc],
+                where=where_asset)
 
         days_diff = days + 5
         query = tab_today.join(
@@ -494,8 +495,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 (tab_today.rate * 100.0 / NullIf(tab_rate2.rate, 0.00) -
                     100.0).as_('percent'),
                 distinct_on=[tab_today.id],
-                order_by=[tab_today.id, tab_rate2.date.desc]
-            )
+                order_by=[tab_today.id, tab_rate2.date.desc])
         return query
 
     @staticmethod
@@ -508,8 +508,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -522,8 +521,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -536,8 +534,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -550,8 +547,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -564,8 +560,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @classmethod
@@ -597,22 +592,22 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         exp = Decimal(Decimal(1) / 10 ** digits_percent)
         asset_id_lst = [x.id for x in assets]
 
-        for x in names:
-            tab_percent = cls.get_percentage_sql(
-                days={
-                    'change_day1': 0,
-                    'change_month1': 30,
-                    'change_month3': 90,
-                    'change_month6': 180,
-                    'change_month12': 365,
-                    }[x],
-                asset_ids=asset_id_lst,
-                )
-            cursor.execute(*tab_percent)
-            records = cursor.fetchall()
+        if asset_id_lst and names:
+            for x in names:
+                tab_percent = cls.get_percentage_sql(
+                    days={
+                        'change_day1': 0,
+                        'change_month1': 30,
+                        'change_month3': 90,
+                        'change_month6': 180,
+                        'change_month12': 365,
+                        }[x],
+                    asset_ids=asset_id_lst)
+                cursor.execute(*tab_percent)
+                records = cursor.fetchall()
 
-            for record in records:
-                result[x][record[0]] = record[3].quantize(exp) \
+                for record in records:
+                    result[x][record[0]] = record[3].quantize(exp) \
                         if record[3] is not None else None
         return result
 
@@ -647,8 +642,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 tab_asset.updtdays,
                 tab_asset.updttime,
                 distinct_on=[tab_asset.id],
-                order_by=[tab_asset.id, tab_rate.date.desc],
-            )
+                order_by=[tab_asset.id, tab_rate.date.desc])
 
         query = tab_date.select(
             tab_date.id,
@@ -660,8 +654,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                     (Extract('dow', tab_date.date) == 6),
                     tab_date.date + Literal(2)),
                 else_=tab_date.date,
-                ) + tab_date.updttime).as_('updttime'),
-            )
+                ) + tab_date.updttime).as_('updttime'))
         return query
 
     @classmethod
@@ -675,8 +668,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         query = tab_updt.select(
                 tab_updt.id,
                 tab_updt.updttime,
-                where=tab_updt.id.in_([x.id for x in assets]),
-            )
+                where=tab_updt.id.in_([x.id for x in assets]))
         cursor.execute(*query)
         records = cursor.fetchall()
 
@@ -700,8 +692,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_updt.select(
                 tab_updt.id,
-                where=Operator(tab_updt.updttime, clause[2]),
-            )
+                where=Operator(tab_updt.updttime, clause[2]))
         return [('id', 'in', query)]
 
     @classmethod
@@ -738,8 +729,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 tab_asset.id,
                 tab_wkn.code.as_('wkn'),
                 tab_secsymb.code.as_('secsymb'),
-                tab_isin.code.as_('isin'),
-            )
+                tab_isin.code.as_('isin'))
         return query
 
     @staticmethod
@@ -763,8 +753,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 condition=tab_templ.id == tab_prod.template
             ).select(
                 tab_templ.name,
-                where=tab_asset.id == table.id
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -777,8 +766,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'wkn'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @staticmethod
@@ -791,8 +779,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'isin'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @staticmethod
@@ -805,8 +792,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'secsymb'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @classmethod
@@ -826,8 +812,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
             ).select(
                 tab_asset.id,
                 where=Operator(field_qu, clause[2]) &
-                (field_qu != None),
-            )
+                (field_qu != DEF_NONE))
 
         return [('id', 'in', query)]
 
@@ -841,20 +826,19 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         cursor = Transaction().connection.cursor()
 
         result = {x: {y.id: None for y in assets} for x in names}
+        if assets:
+            query = cls.get_identifier_sql(tab_asset)
+            query.where = tab_asset.id.in_([x.id for x in assets])
 
-        query = cls.get_identifier_sql(tab_asset)
-        query.where = tab_asset.id.in_([x.id for x in assets])
+            cursor.execute(*query)
+            l1 = cursor.fetchall()
 
-        cursor.execute(*query)
-        l1 = cursor.fetchall()
-
-        for x in l1:
-            (id1, wkn, secsymb, isin) = x
-            r1 = {'wkn': wkn, 'secsymb': secsymb, 'isin': isin}
-
-            for n in names:
-                result[n][id1] = r1[n]
+            for x in l1:
+                (id1, wkn, secsymb, isin) = x
+                r1 = {'wkn': wkn, 'secsymb': secsymb, 'isin': isin}
 
+                for n in names:
+                    result[n][id1] = r1[n]
         return result
 
     def get_rec_name(self, name):
@@ -901,7 +885,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
             if OnlineSource.update_rate(asset):
                 to_run_activities.append(asset)
 
-        if len(to_run_activities) > 0:
+        if to_run_activities:
             cls.after_update_actions(to_run_activities)
 
 # end Asset
@@ -912,11 +896,11 @@ class AssetSourceRel(ModelSQL):
     __name__ = 'investment.asset_source_rel'
 
     source = fields.Many2One(
-        string='Online Source', select=True,
+        string='Online Source',
         required=True, model_name='investment.source',
         ondelete='CASCADE')
     asset = fields.Many2One(
-        string='Asset', select=True,
+        string='Asset',
         required=True, model_name='investment.asset',
         ondelete='CASCADE')
 

const.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# This file is part of the cashbook-module from m-ds.de for Tryton.
+# The COPYRIGHT file at the top level of this repository contains the
+# full copyright notices and license terms.
+
+
+DEF_TRUE = True
+DEF_NONE = None

diagram.py

@@ -60,16 +60,16 @@ class GraphDef(metaclass=PoolMeta):
             return None
 
         if self.scaling == 'alldata':
-            query = [('asset.id', '=', self.asset.id)]
+            query = [('asset', '=', self.asset.id)]
         elif self.scaling == 'view':
             query = [
-                ('asset.id', '=', self.asset.id),
+                ('asset', '=', self.asset.id),
                 ('date', '>=', self.chart.used_start_date()),
                 ('date', '<=', self.chart.used_end_date()),
                 ]
         elif self.scaling == 'six':
             query = [
-                ('asset.id', '=', self.asset.id),
+                ('asset', '=', self.asset.id),
                 ('date', '>=', self.chart.used_start_date() -
                     timedelta(days=180)),
                 ('date', '<=', self.chart.used_end_date()),
@@ -95,7 +95,7 @@ class ChartPoint(metaclass=PoolMeta):
         """
         Rate = Pool().get('investment.rate')
 
-        if keyname is None:
+        if not keyname:
             return None
 
         # check if query is for us
@@ -104,12 +104,12 @@ class ChartPoint(metaclass=PoolMeta):
 
             before = Rate.search([
                 ('date', '<', query_date),
-                ('asset.id', '=', asset_id),
+                ('asset', '=', asset_id),
                 ], limit=1, order=[('date', 'DESC')])
 
             after = Rate.search([
                 ('date', '>', query_date),
-                ('asset.id', '=', asset_id),
+                ('asset', '=', asset_id),
                 ], limit=1, order=[('date', 'ASC')])
 
             if (len(before) == 1) and (len(after) == 1):

import_wiz.py

@@ -84,14 +84,14 @@ class ImportWizard(Wizard):
         pool = Pool()
         ImportWiz = pool.get('investment.imp_wiz', type='wizard')
 
-        if self.start.file_ is not None:
+        if self.start.file_:
             (lines, max_date, min_date) = ImportWiz.read_csv_file(
-                    self.start.file_.decode('utf8'),
-                    dec_divider=self.start.dec_divider,
-                    date_fmt=self.start.date_fmt,
-                    delimiter=self.start.field_delimiter)
+                self.start.file_.decode('utf8'),
+                dec_divider=self.start.dec_divider,
+                date_fmt=self.start.date_fmt,
+                delimiter=self.start.field_delimiter)
 
-            if len(lines) > 0:
+            if lines:
                 ImportWiz.upload_rates(
                     self.start.asset,
                     lines, min_date, max_date)
@@ -173,7 +173,8 @@ class ImportWizard(Wizard):
                         datefmt='dd%sdd' % dec_divider,
                         colnr='2'))
 
-                if isinstance(date_val, date) and isinstance(rate_val, Decimal):
+                if isinstance(date_val, date) and isinstance(
+                        rate_val, Decimal):
                     result.append({'date': date_val, 'rate': rate_val})
 
                     # date range

locale/de.po

@@ -50,6 +50,10 @@ msgctxt "model:ir.message,text:msg_missing_url"
 msgid "URL for the online source '%(oname)s' is missing."
 msgstr "URL für die Onlinequelle '%(oname)s' fehlt."
 
+msgctxt "model:ir.message,text:msg_bug_in_regexquery"
+msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+msgstr "Fehler in Regex-Code des Feldes '%(fname)s': %(errmsg)s [%(code)s]"
+
 
 ##############
 # ir.ui.menu #

locale/en.po

@@ -38,6 +38,10 @@ msgctxt "model:ir.message,text:msg_missing_url"
 msgid "URL for the online source '%(oname)s' is missing."
 msgstr "URL for the online source '%(oname)s' is missing."
 
+msgctxt "model:ir.message,text:msg_bug_in_regexquery"
+msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+msgstr "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+
 msgctxt "model:ir.ui.menu,name:menu_investment"
 msgid "Investment"
 msgstr "Investment"
@@ -294,6 +298,18 @@ msgctxt "selection:investment.asset,updtdays:"
 msgid "Mon - Sun"
 msgstr "Mon - Sun"
 
+msgctxt "field:investment.asset,updturl:"
+msgid "URL"
+msgstr "URL"
+
+msgctxt "help:investment.asset,updturl:"
+msgid "URL for data retrieval."
+msgstr "URL for data retrieval."
+
+msgctxt "field:investment.asset,updturl_enable:"
+msgid "URL required"
+msgstr "URL required"
+
 msgctxt "model:investment.asset_source_rel,name:"
 msgid "Asset Source Relation"
 msgstr "Asset Source Relation"

message.xml

@@ -23,6 +23,9 @@ full copyright notices and license terms. -->
         <record model="ir.message" id="msg_missing_url">
             <field name="text">URL for the online source '%(oname)s' is missing.</field>
         </record>
+        <record model="ir.message" id="msg_bug_in_regexquery">
+            <field name="text">Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]</field>
+        </record>
 
     </data>
 </tryton>

onlinesource.py

@@ -63,16 +63,14 @@ class OnlineSource(ModelSQL, ModelView):
     url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB)
     fixed_url = fields.Boolean(
         string='Fixed URL',
-        states={
-            'invisible': Eval('query_method', '') != 'web',
-        }, depends=DEPENDS_WEB,
+        states={'invisible': Eval('query_method', '') != 'web'},
+        depends=DEPENDS_WEB,
         help='URL must be defined at investment record.')
     nohtml = fields.Boolean(
         string='Remove HTML',
         help='Removes HTML tags before the text is interpreted.',
-        states={
-            'invisible': STATES_WEB['invisible'],
-        }, depends=DEPENDS_WEB)
+        states={'invisible': STATES_WEB['invisible']},
+        depends=DEPENDS_WEB)
     rgxdate = fields.Char(
         string='Date',
         help='Regex code to find the date in the downloaded HTML file.',
@@ -86,14 +84,14 @@ class OnlineSource(ModelSQL, ModelView):
         states=STATES_WEB, depends=DEPENDS_WEB)
     rgxdecimal = fields.Selection(
         string='Decimal Separator',
-        help='Decimal separator for converting the market value into a number.',
+        help='Decimal separator for converting the market ' +
+        'value into a number.',
         selection=sel_rgxdecimal, states=STATES_WEB, depends=DEPENDS_WEB)
     rgxident = fields.Char(
         string='Identifier',
         help='Regex code to find the identifier in the downloaded HTML file.',
-        states={
-            'invisible': STATES_WEB['invisible'],
-        }, depends=DEPENDS_WEB)
+        states={'invisible': STATES_WEB['invisible']},
+        depends=DEPENDS_WEB)
     rgxidtype = fields.Selection(
         string='ID-Type', selection=sel_rgxidtype,
         help='Type of identifier used to validate the result.',
@@ -242,16 +240,13 @@ class OnlineSource(ModelSQL, ModelView):
                 isin=self.isin,
                 nsin=self.nsin,
                 symbol=self.symbol,
-                url=self.url,
-                )
+                url=self.url)
 
     @classmethod
     def get_query_methods(cls):
         """ get list of query-methods
         """
-        return [
-            ('web', gettext('investment.msg_querytype_web')),
-            ]
+        return [('web', gettext('investment.msg_querytype_web'))]
 
     @classmethod
     def set_test_value(cls, record, name, value):
@@ -259,6 +254,15 @@ class OnlineSource(ModelSQL, ModelView):
         """
         pass
 
+    @classmethod
+    def validate(cls, records):
+        """ check regex-code
+        """
+        for record in records:
+            for x in ['rgxdate', 'rgxrate', 'rgxident']:
+                if x:
+                    record.get_regex_result('', x)
+
     @classmethod
     def run_query_method(cls, osource, isin, nsin, symbol, url, debug=False):
         """ run selected query to retrive data
@@ -279,8 +283,7 @@ class OnlineSource(ModelSQL, ModelView):
                 nsin=nsin,
                 symbol=symbol,
                 debug=debug,
-                url=url,
-                )
+                url=url)
 
     def call_online_source(self):
         """ use updated values to call online-source,
@@ -291,7 +294,7 @@ class OnlineSource(ModelSQL, ModelView):
         result = OSourc.run_query_method(
             self, self.isin, self.nsin, self.url,
             self.symbol, debug=True)
-        if result is not None:
+        if result:
             self.text = result.get('text', None)
             self.http_state = result.get('http_state', None)
             self.fnddate = result.get('date', None)
@@ -303,19 +306,17 @@ class OnlineSource(ModelSQL, ModelView):
         """ generate url
         """
         if self.fixed_url is True:
-            if url is None:
+            if not url:
                 raise UserError(gettext(
                     'investment.msg_missing_url',
-                    oname=self.rec_name,
-                    ))
+                    oname=self.rec_name))
             return url
-        else:
-            if self.url:
-                return Template(self.url).substitute({
-                      'isin': isin if isin is not None else '',
-                      'nsin': nsin if nsin is not None else '',
-                      'symbol': symbol if symbol is not None else '',
-                  })
+
+        if self.url:
+            return Template(self.url).substitute({
+                'isin': isin if isin is not None else '',
+                'nsin': nsin if nsin is not None else '',
+                'symbol': symbol if symbol is not None else ''})
 
     @classmethod
     def update_rate(cls, asset):
@@ -334,8 +335,7 @@ class OnlineSource(ModelSQL, ModelView):
                     isin=asset.isin,
                     nsin=asset.wkn,
                     symbol=asset.secsymb,
-                    url=asset.updturl,
-                    )
+                    url=asset.updturl)
 
             if len(updtsource.rgxident or '') > 0:
                 # check result - same code?
@@ -352,15 +352,13 @@ class OnlineSource(ModelSQL, ModelView):
                             'update_rate: got wrong code ' +
                             '"%(wrong)s" - expected "%(exp)s"' % {
                                 'exp': asset_code,
-                                'wrong': code,
-                                })
+                                'wrong': code})
                         continue
 
             to_create = {
                 'date': rate_data.get('date', None),
                 'rate': rate_data.get('rate', None),
-                'asset': asset.id,
-                }
+                'asset': asset.id}
             if (to_create['date'] is not None) and \
                     (to_create['rate'] is not None):
                 # check if exists
@@ -379,13 +377,22 @@ class OnlineSource(ModelSQL, ModelView):
     def get_regex_result(self, html_text, field_name):
         """ run regex on html-text, convert result
         """
+        OSource = Pool().get('investment.source')
+
         rgxcode = getattr(self, field_name) or ''
         if len(rgxcode) == 0:
             return None
 
-        search_result = re.compile(rgxcode).search(html_text)
-        if search_result is None:
-            return None
+        try:
+            search_result = re.compile(rgxcode).search(html_text)
+            if search_result is None:
+                return None
+        except Exception as e1:
+            raise UserError(gettext(
+                'investment.msg_bug_in_regexquery',
+                errmsg=str(e1),
+                fname=getattr(OSource, field_name).string,
+                code=rgxcode))
 
         try:
             result = search_result.group(1)
@@ -426,8 +433,7 @@ class OnlineSource(ModelSQL, ModelView):
                 isin=isin,
                 nsin=nsin,
                 symbol=symbol,
-                url=url,
-                ),
+                url=url),
             allow_redirects=True,
             timeout=5.0)
 
@@ -436,7 +442,7 @@ class OnlineSource(ModelSQL, ModelView):
                 'msg': res1.reason,
             }
 
-        if res1.status_code in [200, 204]:
+        if res1.status_code in [200, 204, 410]:
             html = res1.text
 
             # remove html-tags

rate.py

@@ -4,7 +4,7 @@
 # full copyright notices and license terms.
 
 from trytond.model import (
-        ModelView, ModelSQL, fields, Unique, Check, SymbolMixin)
+        ModelView, ModelSQL, fields, Unique, Check, SymbolMixin, Index)
 from trytond.transaction import Transaction
 from trytond.pool import Pool
 from trytond.pyson import Eval
@@ -15,11 +15,11 @@ class Rate(SymbolMixin, ModelSQL, ModelView):
     __name__ = 'investment.rate'
 
     asset = fields.Many2One(
-        string='Asset', required=True, select=True, ondelete='CASCADE',
+        string='Asset', required=True, ondelete='CASCADE',
         model_name='investment.asset')
-    date = fields.Date(string='Date', required=True, select=True)
+    date = fields.Date(string='Date', required=True)
     rate = fields.Numeric(
-        string='Rate', required=True, select=True,
+        string='Rate', required=True,
         digits=(16, Eval('asset_digits', 4)), depends=['asset_digits'])
 
     asset_digits = fields.Function(fields.Integer(
@@ -46,6 +46,21 @@ class Rate(SymbolMixin, ModelSQL, ModelView):
                 'currency.msg_rate_positive'),
             ]
         cls._order.insert(0, ('date', 'DESC'))
+        cls._sql_indexes.update({
+            Index(
+                t,
+                (t.date, Index.Range(order='DESC'))),
+            Index(
+                t,
+                (t.rate, Index.Range())),
+            Index(
+                t,
+                (t.asset, Index.Equality())),
+            Index(
+                t,
+                (t.asset, Index.Equality()),
+                (t.date, Index.Range(order='DESC'))),
+            })
 
     @classmethod
     def default_date(cls):

setup.py

@@ -39,7 +39,7 @@ with open(path.join(here, 'versiondep.txt'), encoding='utf-8') as f:
         modversion[l2[0]] = {'min': l2[1], 'max': l2[2], 'prefix': l2[3]}
 
 # tryton-version
-major_version = 6
+major_version = 7
 minor_version = 0
 
 requires = ['requests>=2.26', 'html2text']
@@ -91,6 +91,7 @@ setup(
         'License :: OSI Approved :: GNU General Public License (GPL)',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
 
     keywords='tryton investment shares commodities',

sources_def.xml

@@ -71,9 +71,9 @@ full copyright notices and license terms. -->
             <field name="name">www.sbroker.de</field>
             <field name="url">https://www.sbroker.de/sbl/mdaten_analyse/dksuche_a?SEARCH_VALUE=${isin}</field>
             <field name="nohtml" eval="True"/>
-            <field name="rgxdate">\nDatum / Uhrzeit: (\d+\.\d+\.\d+) / \d+:\d+\s+\n</field>
+            <field name="rgxdate">\nDatum\s*(?:\/ Uhrzeit)*: (\d+\.\d+\.\d+)\s*(?:\/ \d+:\d+)*\s*\n</field>
             <field name="rgxdatefmt">%d.%m.%y</field>
-            <field name="rgxrate">Kurs aktuell .* (\d+,\d+)\s+EUR.*\n</field>
+            <field name="rgxrate">(?:Kurs aktuell|Rucknahmepreis)\s+[()\w\./\[\]!]+\s+(\d+,\d+)\s+(EUR|€)</field>
             <field name="rgxdecimal">,</field>
             <field name="rgxident">\nWKN / ISIN: [A-Z,0-9]+ / ([A-Z,0-9]+)\s+\n</field>
             <field name="rgxidtype">isin</field>

tests/__init__.py

@@ -1,17 +1,2 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
-
-import trytond.tests.test_tryton
-import unittest
-
-from .test_module import InvestmentTestCase
-
-
-__all__ = ['suite']
-
-
-def suite():
-    suite = trytond.tests.test_tryton.suite()
-    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(
-        InvestmentTestCase))
-    return suite

tests/source.py

@@ -6,6 +6,7 @@
 from trytond.tests.test_tryton import with_transaction
 from trytond.pool import Pool
 from trytond.transaction import Transaction
+from trytond.exceptions import UserError
 from decimal import Decimal
 from datetime import time, date, datetime
 from unittest.mock import MagicMock
@@ -131,5 +132,54 @@ High 34,87 EUR
             'rgxdate'
             ), date(2022, 3, 14))
 
+    @with_transaction()
+    def test_waitlist_source_check_regex_validate(self):
+        """ create source, check validation of regex-code
+        """
+        pool = Pool()
+        OSource = pool.get('investment.source')
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Date': nothing to repeat " +
+            r"at position 0 \[\*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check date',
+                'rgxdate': '** multiple repeat',
+                'rgxrate': 'rate -- multiple repeat',
+                'rgxident': 'identifiert ** multiple repeat',
+            }])
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Rate': multiple repeat " +
+            r"at position 6 \[rate \*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check rate',
+                'rgxdate': '-- multiple repeat',
+                'rgxrate': 'rate ** multiple repeat',
+                'rgxident': 'identifiert -- multiple repeat',
+            }])
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Identifier': multiple " +
+            r"repeat at position 13 \[identifiert \*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check rgxident',
+                'rgxdate': '-- multiple repeat',
+                'rgxrate': 'rate -- multiple repeat',
+                'rgxident': 'identifiert ** multiple repeat',
+            }])
+
+        OSource.create([{
+            'name': 'Check rgxident',
+            'rgxdate': '-- multiple repeat',
+            'rgxrate': 'rate -- multiple repeat',
+            'rgxident': 'identifiert -- multiple repeat',
+            }])
 
 # end SourceTestCase

tryton.cfg

@@ -1,5 +1,5 @@
 [tryton]
-version=6.0.24
+version=7.0.0
 depends:
     ir
     res

update_wiz.py

@@ -29,7 +29,7 @@ class UpdateSoureWizard(Wizard):
             if OnlineSource.update_rate(asset):
                 to_run_activities.append(asset)
 
-        if len(to_run_activities) > 0:
+        if to_run_activities:
             Asset.after_update_actions(to_run_activities)
 
         return 'end'

versiondep.txt

@@ -1 +1 @@
-diagram;6.0.7;6.0.999;mds
+

view/asset_list.xml

@@ -4,12 +4,12 @@ The COPYRIGHT file at the top level of this repository contains the
 full copyright notices and license terms. -->
 <tree>
     <field name="name" expand="1"/>
-    <field name="change_day1" symbol="change_symbol"/>
-    <field name="change_month1" symbol="change_symbol"/>
-    <field name="change_month3" symbol="change_symbol"/>
-    <field name="change_month6" symbol="change_symbol"/>
-    <field name="date"/>
-    <field name="rate" symbol="asset_symbol"/>
-    <field name="isin"/>
-    <field name="wkn" />
+    <field name="change_day1" optional="0"/>
+    <field name="change_month1" optional="0"/>
+    <field name="change_month3" optional="0"/>
+    <field name="change_month6" optional="0"/>
+    <field name="date" optional="0"/>
+    <field name="rate" symbol="asset_symbol" optional="0"/>
+    <field name="isin" optional="0"/>
+    <field name="wkn" optional="0"/>
 </tree>