optimize code

asset.py

@@ -46,12 +46,9 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
     uom = fields.Many2One(
         string='UOM', required=True, model_name='product.uom',
         ondelete='RESTRICT',
-        states={
-            'readonly': ~Bool(Eval('product')),
-        },
-        domain=[
-            ('category', '=', Eval('product_uom')),
-        ], depends=['product_uom', 'product'])
+        states={'readonly': ~Bool(Eval('product'))},
+        domain=[('category', '=', Eval('product_uom'))],
+        depends=['product_uom', 'product'])
     symbol = fields.Function(fields.Char(
         string='UOM', readonly=True), 'get_name_symbol',
         searcher='search_uom_symbol')
@@ -74,9 +71,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         model_name='currency.currency', ondelete='RESTRICT')
     currency_digits = fields.Integer(
         string='Digits', required=True,
-        domain=[
-            ('currency_digits', '>=', 0),
-            ('currency_digits', '<=', 6)])
+        domain=[('currency_digits', '>=', 0), ('currency_digits', '<=', 6)])
 
     wkn = fields.Function(fields.Char(
         string='NSIN', readonly=True,
@@ -112,9 +107,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         string='Select days', required=True, selection=sel_updtdays)
     updttime = fields.Time(
         string='Time',
-        states={
-            'readonly': ~Bool(Eval('updtsources')),
-        }, depends=['updtsources'])
+        states={'readonly': ~Bool(Eval('updtsources'))},
+        depends=['updtsources'])
     nextupdate = fields.Function(fields.DateTime(
         string='Next Update', readonly=True),
         'get_nextupdates', searcher='search_nextupdate')
@@ -201,7 +195,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                     'source': x[1],
                 } for x in records]
 
-            if len(to_create) > 0:
+            if to_create:
                 AssetSourceRel.create(to_create)
             asset_table.drop_column('updtsource')
 
@@ -262,7 +256,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
     def on_change_updtsources(self):
         """ clear time-fields
         """
-        if len(self.updtsources) == 0:
+        if not self.updtsources:
             self.updttime = None
         else:
             self.updttime = time(11, 30)
@@ -423,8 +417,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.id,
-                where=Operator(tab_query.date, clause[2]),
-            )
+                where=Operator(tab_query.date, clause[2]))
         return [('id', 'in', query)]
 
     @classmethod
@@ -436,8 +429,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.id,
-                where=Operator(tab_query.rate, clause[2]),
-            )
+                where=Operator(tab_query.rate, clause[2]))
         return [('id', 'in', query)]
 
     @staticmethod
@@ -449,8 +441,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.date,
-                where=tab_query.id == table.id,
-            )
+                where=tab_query.id == table.id)
         return [query]
 
     @staticmethod
@@ -462,8 +453,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_query.select(
                 tab_query.rate,
-                where=tab_query.id == table.id,
-            )
+                where=tab_query.id == table.id)
         return [query]
 
     @classmethod
@@ -489,8 +479,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 tab_rate1.rate,
                 distinct_on=[tab_rate1.asset],
                 order_by=[tab_rate1.asset, tab_rate1.date.desc],
-                where=where_asset,
-            )
+                where=where_asset)
 
         days_diff = days + 5
         query = tab_today.join(
@@ -506,8 +495,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 (tab_today.rate * 100.0 / NullIf(tab_rate2.rate, 0.00) -
                     100.0).as_('percent'),
                 distinct_on=[tab_today.id],
-                order_by=[tab_today.id, tab_rate2.date.desc]
-            )
+                order_by=[tab_today.id, tab_rate2.date.desc])
         return query
 
     @staticmethod
@@ -520,8 +508,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -534,8 +521,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -548,8 +534,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -562,8 +547,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -576,8 +560,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_asset.select(
                 tab_asset.percent,
-                where=tab_asset.id == table.id,
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @classmethod
@@ -619,8 +602,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                         'change_month6': 180,
                         'change_month12': 365,
                         }[x],
-                    asset_ids=asset_id_lst,
-                    )
+                    asset_ids=asset_id_lst)
                 cursor.execute(*tab_percent)
                 records = cursor.fetchall()
 
@@ -660,8 +642,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 tab_asset.updtdays,
                 tab_asset.updttime,
                 distinct_on=[tab_asset.id],
-                order_by=[tab_asset.id, tab_rate.date.desc],
-            )
+                order_by=[tab_asset.id, tab_rate.date.desc])
 
         query = tab_date.select(
             tab_date.id,
@@ -673,8 +654,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                     (Extract('dow', tab_date.date) == 6),
                     tab_date.date + Literal(2)),
                 else_=tab_date.date,
-                ) + tab_date.updttime).as_('updttime'),
-            )
+                ) + tab_date.updttime).as_('updttime'))
         return query
 
     @classmethod
@@ -688,8 +668,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
         query = tab_updt.select(
                 tab_updt.id,
                 tab_updt.updttime,
-                where=tab_updt.id.in_([x.id for x in assets]),
-            )
+                where=tab_updt.id.in_([x.id for x in assets]))
         cursor.execute(*query)
         records = cursor.fetchall()
 
@@ -713,8 +692,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_updt.select(
                 tab_updt.id,
-                where=Operator(tab_updt.updttime, clause[2]),
-            )
+                where=Operator(tab_updt.updttime, clause[2]))
         return [('id', 'in', query)]
 
     @classmethod
@@ -751,8 +729,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 tab_asset.id,
                 tab_wkn.code.as_('wkn'),
                 tab_secsymb.code.as_('secsymb'),
-                tab_isin.code.as_('isin'),
-            )
+                tab_isin.code.as_('isin'))
         return query
 
     @staticmethod
@@ -776,8 +753,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
                 condition=tab_templ.id == tab_prod.template
             ).select(
                 tab_templ.name,
-                where=tab_asset.id == table.id
-            )
+                where=tab_asset.id == table.id)
         return [query]
 
     @staticmethod
@@ -790,8 +766,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'wkn'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @staticmethod
@@ -804,8 +779,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'isin'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @staticmethod
@@ -818,8 +792,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
 
         query = tab_ids.select(
                 getattr(tab_ids, 'secsymb'),
-                where=tab_ids.id == table.id,
-            )
+                where=tab_ids.id == table.id)
         return [query]
 
     @classmethod
@@ -839,8 +812,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
             ).select(
                 tab_asset.id,
                 where=Operator(field_qu, clause[2]) &
-                (field_qu != DEF_NONE),
-            )
+                (field_qu != DEF_NONE))
 
         return [('id', 'in', query)]
 
@@ -913,7 +885,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView):
             if OnlineSource.update_rate(asset):
                 to_run_activities.append(asset)
 
-        if len(to_run_activities) > 0:
+        if to_run_activities:
             cls.after_update_actions(to_run_activities)
 
 # end Asset

diagram.py

@@ -95,7 +95,7 @@ class ChartPoint(metaclass=PoolMeta):
         """
         Rate = Pool().get('investment.rate')
 
-        if keyname is None:
+        if not keyname:
             return None
 
         # check if query is for us

import_wiz.py

@@ -84,14 +84,14 @@ class ImportWizard(Wizard):
         pool = Pool()
         ImportWiz = pool.get('investment.imp_wiz', type='wizard')
 
-        if self.start.file_ is not None:
+        if self.start.file_:
             (lines, max_date, min_date) = ImportWiz.read_csv_file(
                 self.start.file_.decode('utf8'),
                 dec_divider=self.start.dec_divider,
                 date_fmt=self.start.date_fmt,
                 delimiter=self.start.field_delimiter)
 
-            if len(lines) > 0:
+            if lines:
                 ImportWiz.upload_rates(
                     self.start.asset,
                     lines, min_date, max_date)

locale/de.po

@@ -50,6 +50,10 @@ msgctxt "model:ir.message,text:msg_missing_url"
 msgid "URL for the online source '%(oname)s' is missing."
 msgstr "URL für die Onlinequelle '%(oname)s' fehlt."
 
+msgctxt "model:ir.message,text:msg_bug_in_regexquery"
+msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+msgstr "Fehler in Regex-Code des Feldes '%(fname)s': %(errmsg)s [%(code)s]"
+
 
 ##############
 # ir.ui.menu #

locale/en.po

@@ -38,6 +38,10 @@ msgctxt "model:ir.message,text:msg_missing_url"
 msgid "URL for the online source '%(oname)s' is missing."
 msgstr "URL for the online source '%(oname)s' is missing."
 
+msgctxt "model:ir.message,text:msg_bug_in_regexquery"
+msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+msgstr "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]"
+
 msgctxt "model:ir.ui.menu,name:menu_investment"
 msgid "Investment"
 msgstr "Investment"
@@ -294,6 +298,18 @@ msgctxt "selection:investment.asset,updtdays:"
 msgid "Mon - Sun"
 msgstr "Mon - Sun"
 
+msgctxt "field:investment.asset,updturl:"
+msgid "URL"
+msgstr "URL"
+
+msgctxt "help:investment.asset,updturl:"
+msgid "URL for data retrieval."
+msgstr "URL for data retrieval."
+
+msgctxt "field:investment.asset,updturl_enable:"
+msgid "URL required"
+msgstr "URL required"
+
 msgctxt "model:investment.asset_source_rel,name:"
 msgid "Asset Source Relation"
 msgstr "Asset Source Relation"

message.xml

@@ -23,6 +23,9 @@ full copyright notices and license terms. -->
         <record model="ir.message" id="msg_missing_url">
             <field name="text">URL for the online source '%(oname)s' is missing.</field>
         </record>
+        <record model="ir.message" id="msg_bug_in_regexquery">
+            <field name="text">Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]</field>
+        </record>
 
     </data>
 </tryton>

onlinesource.py

@@ -63,16 +63,14 @@ class OnlineSource(ModelSQL, ModelView):
     url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB)
     fixed_url = fields.Boolean(
         string='Fixed URL',
-        states={
-            'invisible': Eval('query_method', '') != 'web',
-        }, depends=DEPENDS_WEB,
+        states={'invisible': Eval('query_method', '') != 'web'},
+        depends=DEPENDS_WEB,
         help='URL must be defined at investment record.')
     nohtml = fields.Boolean(
         string='Remove HTML',
         help='Removes HTML tags before the text is interpreted.',
-        states={
-            'invisible': STATES_WEB['invisible'],
-        }, depends=DEPENDS_WEB)
+        states={'invisible': STATES_WEB['invisible']},
+        depends=DEPENDS_WEB)
     rgxdate = fields.Char(
         string='Date',
         help='Regex code to find the date in the downloaded HTML file.',
@@ -92,9 +90,8 @@ class OnlineSource(ModelSQL, ModelView):
     rgxident = fields.Char(
         string='Identifier',
         help='Regex code to find the identifier in the downloaded HTML file.',
-        states={
-            'invisible': STATES_WEB['invisible'],
-        }, depends=DEPENDS_WEB)
+        states={'invisible': STATES_WEB['invisible']},
+        depends=DEPENDS_WEB)
     rgxidtype = fields.Selection(
         string='ID-Type', selection=sel_rgxidtype,
         help='Type of identifier used to validate the result.',
@@ -243,16 +240,13 @@ class OnlineSource(ModelSQL, ModelView):
                 isin=self.isin,
                 nsin=self.nsin,
                 symbol=self.symbol,
-                url=self.url,
-                )
+                url=self.url)
 
     @classmethod
     def get_query_methods(cls):
         """ get list of query-methods
         """
-        return [
-            ('web', gettext('investment.msg_querytype_web')),
-            ]
+        return [('web', gettext('investment.msg_querytype_web'))]
 
     @classmethod
     def set_test_value(cls, record, name, value):
@@ -260,6 +254,15 @@ class OnlineSource(ModelSQL, ModelView):
         """
         pass
 
+    @classmethod
+    def validate(cls, records):
+        """ check regex-code
+        """
+        for record in records:
+            for x in ['rgxdate', 'rgxrate', 'rgxident']:
+                if x:
+                    record.get_regex_result('', x)
+
     @classmethod
     def run_query_method(cls, osource, isin, nsin, symbol, url, debug=False):
         """ run selected query to retrive data
@@ -280,8 +283,7 @@ class OnlineSource(ModelSQL, ModelView):
                 nsin=nsin,
                 symbol=symbol,
                 debug=debug,
-                url=url,
-                )
+                url=url)
 
     def call_online_source(self):
         """ use updated values to call online-source,
@@ -292,7 +294,7 @@ class OnlineSource(ModelSQL, ModelView):
         result = OSourc.run_query_method(
             self, self.isin, self.nsin, self.url,
             self.symbol, debug=True)
-        if result is not None:
+        if result:
             self.text = result.get('text', None)
             self.http_state = result.get('http_state', None)
             self.fnddate = result.get('date', None)
@@ -304,19 +306,17 @@ class OnlineSource(ModelSQL, ModelView):
         """ generate url
         """
         if self.fixed_url is True:
-            if url is None:
+            if not url:
                 raise UserError(gettext(
                     'investment.msg_missing_url',
-                    oname=self.rec_name,
-                    ))
+                    oname=self.rec_name))
             return url
-        else:
+
         if self.url:
             return Template(self.url).substitute({
                 'isin': isin if isin is not None else '',
                 'nsin': nsin if nsin is not None else '',
-                      'symbol': symbol if symbol is not None else '',
-                  })
+                'symbol': symbol if symbol is not None else ''})
 
     @classmethod
     def update_rate(cls, asset):
@@ -335,8 +335,7 @@ class OnlineSource(ModelSQL, ModelView):
                     isin=asset.isin,
                     nsin=asset.wkn,
                     symbol=asset.secsymb,
-                    url=asset.updturl,
-                    )
+                    url=asset.updturl)
 
             if len(updtsource.rgxident or '') > 0:
                 # check result - same code?
@@ -353,15 +352,13 @@ class OnlineSource(ModelSQL, ModelView):
                             'update_rate: got wrong code ' +
                             '"%(wrong)s" - expected "%(exp)s"' % {
                                 'exp': asset_code,
-                                'wrong': code,
-                                })
+                                'wrong': code})
                         continue
 
             to_create = {
                 'date': rate_data.get('date', None),
                 'rate': rate_data.get('rate', None),
-                'asset': asset.id,
-                }
+                'asset': asset.id}
             if (to_create['date'] is not None) and \
                     (to_create['rate'] is not None):
                 # check if exists
@@ -380,13 +377,22 @@ class OnlineSource(ModelSQL, ModelView):
     def get_regex_result(self, html_text, field_name):
         """ run regex on html-text, convert result
         """
+        OSource = Pool().get('investment.source')
+
         rgxcode = getattr(self, field_name) or ''
         if len(rgxcode) == 0:
             return None
 
+        try:
             search_result = re.compile(rgxcode).search(html_text)
             if search_result is None:
                 return None
+        except Exception as e1:
+            raise UserError(gettext(
+                'investment.msg_bug_in_regexquery',
+                errmsg=str(e1),
+                fname=getattr(OSource, field_name).string,
+                code=rgxcode))
 
         try:
             result = search_result.group(1)
@@ -427,8 +433,7 @@ class OnlineSource(ModelSQL, ModelView):
                 isin=isin,
                 nsin=nsin,
                 symbol=symbol,
-                url=url,
-                ),
+                url=url),
             allow_redirects=True,
             timeout=5.0)
 
@@ -437,7 +442,7 @@ class OnlineSource(ModelSQL, ModelView):
                 'msg': res1.reason,
             }
 
-        if res1.status_code in [200, 204]:
+        if res1.status_code in [200, 204, 410]:
             html = res1.text
 
             # remove html-tags

sources_def.xml

@@ -71,9 +71,9 @@ full copyright notices and license terms. -->
             <field name="name">www.sbroker.de</field>
             <field name="url">https://www.sbroker.de/sbl/mdaten_analyse/dksuche_a?SEARCH_VALUE=${isin}</field>
             <field name="nohtml" eval="True"/>
-            <field name="rgxdate">\nDatum / Uhrzeit: (\d+\.\d+\.\d+) / \d+:\d+\s+\n</field>
+            <field name="rgxdate">\nDatum\s*(?:\/ Uhrzeit)*: (\d+\.\d+\.\d+)\s*(?:\/ \d+:\d+)*\s*\n</field>
             <field name="rgxdatefmt">%d.%m.%y</field>
-            <field name="rgxrate">Kurs aktuell .* (\d+,\d+)\s+EUR.*\n</field>
+            <field name="rgxrate">(?:Kurs aktuell|Rucknahmepreis)\s+[()\w\./\[\]!]+\s+(\d+,\d+)\s+(EUR|€)</field>
             <field name="rgxdecimal">,</field>
             <field name="rgxident">\nWKN / ISIN: [A-Z,0-9]+ / ([A-Z,0-9]+)\s+\n</field>
             <field name="rgxidtype">isin</field>

tests/source.py

@@ -6,6 +6,7 @@
 from trytond.tests.test_tryton import with_transaction
 from trytond.pool import Pool
 from trytond.transaction import Transaction
+from trytond.exceptions import UserError
 from decimal import Decimal
 from datetime import time, date, datetime
 from unittest.mock import MagicMock
@@ -131,5 +132,54 @@ High 34,87 EUR
             'rgxdate'
             ), date(2022, 3, 14))
 
+    @with_transaction()
+    def test_waitlist_source_check_regex_validate(self):
+        """ create source, check validation of regex-code
+        """
+        pool = Pool()
+        OSource = pool.get('investment.source')
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Date': nothing to repeat " +
+            r"at position 0 \[\*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check date',
+                'rgxdate': '** multiple repeat',
+                'rgxrate': 'rate -- multiple repeat',
+                'rgxident': 'identifiert ** multiple repeat',
+            }])
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Rate': multiple repeat " +
+            r"at position 6 \[rate \*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check rate',
+                'rgxdate': '-- multiple repeat',
+                'rgxrate': 'rate ** multiple repeat',
+                'rgxident': 'identifiert -- multiple repeat',
+            }])
+
+        self.assertRaisesRegex(
+            UserError,
+            r"Error in regex code of field 'Identifier': multiple " +
+            r"repeat at position 13 \[identifiert \*+ multiple repeat\]",
+            OSource.create,
+            [{
+                'name': 'Check rgxident',
+                'rgxdate': '-- multiple repeat',
+                'rgxrate': 'rate -- multiple repeat',
+                'rgxident': 'identifiert ** multiple repeat',
+            }])
+
+        OSource.create([{
+            'name': 'Check rgxident',
+            'rgxdate': '-- multiple repeat',
+            'rgxrate': 'rate -- multiple repeat',
+            'rgxident': 'identifiert -- multiple repeat',
+            }])
 
 # end SourceTestCase

update_wiz.py

@@ -29,7 +29,7 @@ class UpdateSoureWizard(Wizard):
             if OnlineSource.update_rate(asset):
                 to_run_activities.append(asset)
 
-        if len(to_run_activities) > 0:
+        if to_run_activities:
             Asset.after_update_actions(to_run_activities)
 
         return 'end'

view/asset_list.xml

@@ -4,12 +4,12 @@ The COPYRIGHT file at the top level of this repository contains the
 full copyright notices and license terms. -->
 <tree>
     <field name="name" expand="1"/>
-    <field name="change_day1"/>
-    <field name="change_month1"/>
-    <field name="change_month3"/>
-    <field name="change_month6"/>
-    <field name="date"/>
-    <field name="rate" symbol="asset_symbol"/>
-    <field name="isin"/>
-    <field name="wkn" />
+    <field name="change_day1" optional="0"/>
+    <field name="change_month1" optional="0"/>
+    <field name="change_month3" optional="0"/>
+    <field name="change_month6" optional="0"/>
+    <field name="date" optional="0"/>
+    <field name="rate" symbol="asset_symbol" optional="0"/>
+    <field name="isin" optional="0"/>
+    <field name="wkn" optional="0"/>
 </tree>