diff --git a/asset.py b/asset.py index dd62caa..578de36 100644 --- a/asset.py +++ b/asset.py @@ -32,7 +32,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): name = fields.Function(fields.Char( string='Name', readonly=True), - 'get_name_symbol', searcher='search_rec_name') + 'get_name_symbol', searcher='search_rec_name') company = fields.Many2One( string='Company', model_name='company.company', required=True, ondelete="RESTRICT") @@ -46,12 +46,9 @@ class Asset(SymbolMixin, ModelSQL, ModelView): uom = fields.Many2One( string='UOM', required=True, model_name='product.uom', ondelete='RESTRICT', - states={ - 'readonly': ~Bool(Eval('product')), - }, - domain=[ - ('category', '=', Eval('product_uom')), - ], depends=['product_uom', 'product']) + states={'readonly': ~Bool(Eval('product'))}, + domain=[('category', '=', Eval('product_uom'))], + depends=['product_uom', 'product']) symbol = fields.Function(fields.Char( string='UOM', readonly=True), 'get_name_symbol', searcher='search_uom_symbol') @@ -74,9 +71,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): model_name='currency.currency', ondelete='RESTRICT') currency_digits = fields.Integer( string='Digits', required=True, - domain=[ - ('currency_digits', '>=', 0), - ('currency_digits', '<=', 6)]) + domain=[('currency_digits', '>=', 0), ('currency_digits', '<=', 6)]) wkn = fields.Function(fields.Char( string='NSIN', readonly=True, @@ -112,9 +107,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView): string='Select days', required=True, selection=sel_updtdays) updttime = fields.Time( string='Time', - states={ - 'readonly': ~Bool(Eval('updtsources')), - }, depends=['updtsources']) + states={'readonly': ~Bool(Eval('updtsources'))}, + depends=['updtsources']) nextupdate = fields.Function(fields.DateTime( string='Next Update', readonly=True), 'get_nextupdates', searcher='search_nextupdate') @@ -201,7 +195,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): 'source': x[1], } for x in records] - if len(to_create) > 0: + if to_create: AssetSourceRel.create(to_create) asset_table.drop_column('updtsource') @@ -262,7 +256,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): def on_change_updtsources(self): """ clear time-fields """ - if len(self.updtsources) == 0: + if not self.updtsources: self.updttime = None else: self.updttime = time(11, 30) @@ -423,8 +417,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.id, - where=Operator(tab_query.date, clause[2]), - ) + where=Operator(tab_query.date, clause[2])) return [('id', 'in', query)] @classmethod @@ -436,8 +429,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.id, - where=Operator(tab_query.rate, clause[2]), - ) + where=Operator(tab_query.rate, clause[2])) return [('id', 'in', query)] @staticmethod @@ -449,8 +441,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.date, - where=tab_query.id == table.id, - ) + where=tab_query.id == table.id) return [query] @staticmethod @@ -462,8 +453,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.rate, - where=tab_query.id == table.id, - ) + where=tab_query.id == table.id) return [query] @classmethod @@ -489,8 +479,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_rate1.rate, distinct_on=[tab_rate1.asset], order_by=[tab_rate1.asset, tab_rate1.date.desc], - where=where_asset, - ) + where=where_asset) days_diff = days + 5 query = tab_today.join( @@ -506,8 +495,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): (tab_today.rate * 100.0 / NullIf(tab_rate2.rate, 0.00) - 100.0).as_('percent'), distinct_on=[tab_today.id], - order_by=[tab_today.id, tab_rate2.date.desc] - ) + order_by=[tab_today.id, tab_rate2.date.desc]) return query @staticmethod @@ -520,8 +508,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -534,8 +521,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -548,8 +534,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -562,8 +547,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -576,8 +560,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @classmethod @@ -619,14 +602,13 @@ class Asset(SymbolMixin, ModelSQL, ModelView): 'change_month6': 180, 'change_month12': 365, }[x], - asset_ids=asset_id_lst, - ) + asset_ids=asset_id_lst) cursor.execute(*tab_percent) records = cursor.fetchall() for record in records: result[x][record[0]] = record[3].quantize(exp) \ - if record[3] is not None else None + if record[3] is not None else None return result @classmethod @@ -660,8 +642,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_asset.updtdays, tab_asset.updttime, distinct_on=[tab_asset.id], - order_by=[tab_asset.id, tab_rate.date.desc], - ) + order_by=[tab_asset.id, tab_rate.date.desc]) query = tab_date.select( tab_date.id, @@ -673,8 +654,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): (Extract('dow', tab_date.date) == 6), tab_date.date + Literal(2)), else_=tab_date.date, - ) + tab_date.updttime).as_('updttime'), - ) + ) + tab_date.updttime).as_('updttime')) return query @classmethod @@ -688,8 +668,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_updt.select( tab_updt.id, tab_updt.updttime, - where=tab_updt.id.in_([x.id for x in assets]), - ) + where=tab_updt.id.in_([x.id for x in assets])) cursor.execute(*query) records = cursor.fetchall() @@ -713,8 +692,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_updt.select( tab_updt.id, - where=Operator(tab_updt.updttime, clause[2]), - ) + where=Operator(tab_updt.updttime, clause[2])) return [('id', 'in', query)] @classmethod @@ -751,8 +729,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_asset.id, tab_wkn.code.as_('wkn'), tab_secsymb.code.as_('secsymb'), - tab_isin.code.as_('isin'), - ) + tab_isin.code.as_('isin')) return query @staticmethod @@ -776,8 +753,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): condition=tab_templ.id == tab_prod.template ).select( tab_templ.name, - where=tab_asset.id == table.id - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -790,8 +766,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'wkn'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @staticmethod @@ -804,8 +779,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'isin'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @staticmethod @@ -818,8 +792,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'secsymb'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @classmethod @@ -839,8 +812,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): ).select( tab_asset.id, where=Operator(field_qu, clause[2]) & - (field_qu != DEF_NONE), - ) + (field_qu != DEF_NONE)) return [('id', 'in', query)] @@ -913,7 +885,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): if OnlineSource.update_rate(asset): to_run_activities.append(asset) - if len(to_run_activities) > 0: + if to_run_activities: cls.after_update_actions(to_run_activities) # end Asset diff --git a/diagram.py b/diagram.py index 08c7cab..1d654e9 100644 --- a/diagram.py +++ b/diagram.py @@ -95,7 +95,7 @@ class ChartPoint(metaclass=PoolMeta): """ Rate = Pool().get('investment.rate') - if keyname is None: + if not keyname: return None # check if query is for us diff --git a/import_wiz.py b/import_wiz.py index 4722557..5a4d097 100644 --- a/import_wiz.py +++ b/import_wiz.py @@ -84,14 +84,14 @@ class ImportWizard(Wizard): pool = Pool() ImportWiz = pool.get('investment.imp_wiz', type='wizard') - if self.start.file_ is not None: + if self.start.file_: (lines, max_date, min_date) = ImportWiz.read_csv_file( - self.start.file_.decode('utf8'), - dec_divider=self.start.dec_divider, - date_fmt=self.start.date_fmt, - delimiter=self.start.field_delimiter) + self.start.file_.decode('utf8'), + dec_divider=self.start.dec_divider, + date_fmt=self.start.date_fmt, + delimiter=self.start.field_delimiter) - if len(lines) > 0: + if lines: ImportWiz.upload_rates( self.start.asset, lines, min_date, max_date) diff --git a/locale/de.po b/locale/de.po index 5d55abe..2e027be 100644 --- a/locale/de.po +++ b/locale/de.po @@ -50,6 +50,10 @@ msgctxt "model:ir.message,text:msg_missing_url" msgid "URL for the online source '%(oname)s' is missing." msgstr "URL für die Onlinequelle '%(oname)s' fehlt." +msgctxt "model:ir.message,text:msg_bug_in_regexquery" +msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" +msgstr "Fehler in Regex-Code des Feldes '%(fname)s': %(errmsg)s [%(code)s]" + ############## # ir.ui.menu # diff --git a/locale/en.po b/locale/en.po index 7e0923e..7d76175 100644 --- a/locale/en.po +++ b/locale/en.po @@ -38,6 +38,10 @@ msgctxt "model:ir.message,text:msg_missing_url" msgid "URL for the online source '%(oname)s' is missing." msgstr "URL for the online source '%(oname)s' is missing." +msgctxt "model:ir.message,text:msg_bug_in_regexquery" +msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" +msgstr "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" + msgctxt "model:ir.ui.menu,name:menu_investment" msgid "Investment" msgstr "Investment" @@ -294,6 +298,18 @@ msgctxt "selection:investment.asset,updtdays:" msgid "Mon - Sun" msgstr "Mon - Sun" +msgctxt "field:investment.asset,updturl:" +msgid "URL" +msgstr "URL" + +msgctxt "help:investment.asset,updturl:" +msgid "URL for data retrieval." +msgstr "URL for data retrieval." + +msgctxt "field:investment.asset,updturl_enable:" +msgid "URL required" +msgstr "URL required" + msgctxt "model:investment.asset_source_rel,name:" msgid "Asset Source Relation" msgstr "Asset Source Relation" diff --git a/message.xml b/message.xml index 2ef9571..2d681ef 100644 --- a/message.xml +++ b/message.xml @@ -23,6 +23,9 @@ full copyright notices and license terms. --> URL for the online source '%(oname)s' is missing. + + Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s] + diff --git a/onlinesource.py b/onlinesource.py index a74696a..abda8b4 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -63,16 +63,14 @@ class OnlineSource(ModelSQL, ModelView): url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB) fixed_url = fields.Boolean( string='Fixed URL', - states={ - 'invisible': Eval('query_method', '') != 'web', - }, depends=DEPENDS_WEB, + states={'invisible': Eval('query_method', '') != 'web'}, + depends=DEPENDS_WEB, help='URL must be defined at investment record.') nohtml = fields.Boolean( string='Remove HTML', help='Removes HTML tags before the text is interpreted.', - states={ - 'invisible': STATES_WEB['invisible'], - }, depends=DEPENDS_WEB) + states={'invisible': STATES_WEB['invisible']}, + depends=DEPENDS_WEB) rgxdate = fields.Char( string='Date', help='Regex code to find the date in the downloaded HTML file.', @@ -92,9 +90,8 @@ class OnlineSource(ModelSQL, ModelView): rgxident = fields.Char( string='Identifier', help='Regex code to find the identifier in the downloaded HTML file.', - states={ - 'invisible': STATES_WEB['invisible'], - }, depends=DEPENDS_WEB) + states={'invisible': STATES_WEB['invisible']}, + depends=DEPENDS_WEB) rgxidtype = fields.Selection( string='ID-Type', selection=sel_rgxidtype, help='Type of identifier used to validate the result.', @@ -243,16 +240,13 @@ class OnlineSource(ModelSQL, ModelView): isin=self.isin, nsin=self.nsin, symbol=self.symbol, - url=self.url, - ) + url=self.url) @classmethod def get_query_methods(cls): """ get list of query-methods """ - return [ - ('web', gettext('investment.msg_querytype_web')), - ] + return [('web', gettext('investment.msg_querytype_web'))] @classmethod def set_test_value(cls, record, name, value): @@ -260,6 +254,15 @@ class OnlineSource(ModelSQL, ModelView): """ pass + @classmethod + def validate(cls, records): + """ check regex-code + """ + for record in records: + for x in ['rgxdate', 'rgxrate', 'rgxident']: + if x: + record.get_regex_result('', x) + @classmethod def run_query_method(cls, osource, isin, nsin, symbol, url, debug=False): """ run selected query to retrive data @@ -280,8 +283,7 @@ class OnlineSource(ModelSQL, ModelView): nsin=nsin, symbol=symbol, debug=debug, - url=url, - ) + url=url) def call_online_source(self): """ use updated values to call online-source, @@ -292,7 +294,7 @@ class OnlineSource(ModelSQL, ModelView): result = OSourc.run_query_method( self, self.isin, self.nsin, self.url, self.symbol, debug=True) - if result is not None: + if result: self.text = result.get('text', None) self.http_state = result.get('http_state', None) self.fnddate = result.get('date', None) @@ -304,19 +306,17 @@ class OnlineSource(ModelSQL, ModelView): """ generate url """ if self.fixed_url is True: - if url is None: + if not url: raise UserError(gettext( 'investment.msg_missing_url', - oname=self.rec_name, - )) + oname=self.rec_name)) return url - else: - if self.url: - return Template(self.url).substitute({ - 'isin': isin if isin is not None else '', - 'nsin': nsin if nsin is not None else '', - 'symbol': symbol if symbol is not None else '', - }) + + if self.url: + return Template(self.url).substitute({ + 'isin': isin if isin is not None else '', + 'nsin': nsin if nsin is not None else '', + 'symbol': symbol if symbol is not None else ''}) @classmethod def update_rate(cls, asset): @@ -335,8 +335,7 @@ class OnlineSource(ModelSQL, ModelView): isin=asset.isin, nsin=asset.wkn, symbol=asset.secsymb, - url=asset.updturl, - ) + url=asset.updturl) if len(updtsource.rgxident or '') > 0: # check result - same code? @@ -353,15 +352,13 @@ class OnlineSource(ModelSQL, ModelView): 'update_rate: got wrong code ' + '"%(wrong)s" - expected "%(exp)s"' % { 'exp': asset_code, - 'wrong': code, - }) + 'wrong': code}) continue to_create = { 'date': rate_data.get('date', None), 'rate': rate_data.get('rate', None), - 'asset': asset.id, - } + 'asset': asset.id} if (to_create['date'] is not None) and \ (to_create['rate'] is not None): # check if exists @@ -380,13 +377,22 @@ class OnlineSource(ModelSQL, ModelView): def get_regex_result(self, html_text, field_name): """ run regex on html-text, convert result """ + OSource = Pool().get('investment.source') + rgxcode = getattr(self, field_name) or '' if len(rgxcode) == 0: return None - search_result = re.compile(rgxcode).search(html_text) - if search_result is None: - return None + try: + search_result = re.compile(rgxcode).search(html_text) + if search_result is None: + return None + except Exception as e1: + raise UserError(gettext( + 'investment.msg_bug_in_regexquery', + errmsg=str(e1), + fname=getattr(OSource, field_name).string, + code=rgxcode)) try: result = search_result.group(1) @@ -427,8 +433,7 @@ class OnlineSource(ModelSQL, ModelView): isin=isin, nsin=nsin, symbol=symbol, - url=url, - ), + url=url), allow_redirects=True, timeout=5.0) @@ -437,7 +442,7 @@ class OnlineSource(ModelSQL, ModelView): 'msg': res1.reason, } - if res1.status_code in [200, 204]: + if res1.status_code in [200, 204, 410]: html = res1.text # remove html-tags diff --git a/sources_def.xml b/sources_def.xml index 1f49541..ff5959a 100644 --- a/sources_def.xml +++ b/sources_def.xml @@ -71,9 +71,9 @@ full copyright notices and license terms. --> www.sbroker.de https://www.sbroker.de/sbl/mdaten_analyse/dksuche_a?SEARCH_VALUE=${isin} - \nDatum / Uhrzeit: (\d+\.\d+\.\d+) / \d+:\d+\s+\n + \nDatum\s*(?:\/ Uhrzeit)*: (\d+\.\d+\.\d+)\s*(?:\/ \d+:\d+)*\s*\n %d.%m.%y - Kurs aktuell .* (\d+,\d+)\s+EUR.*\n + (?:Kurs aktuell|Rucknahmepreis)\s+[()\w\./\[\]!]+\s+(\d+,\d+)\s+(EUR|€) , \nWKN / ISIN: [A-Z,0-9]+ / ([A-Z,0-9]+)\s+\n isin diff --git a/tests/source.py b/tests/source.py index f784e4c..43ac22b 100644 --- a/tests/source.py +++ b/tests/source.py @@ -6,6 +6,7 @@ from trytond.tests.test_tryton import with_transaction from trytond.pool import Pool from trytond.transaction import Transaction +from trytond.exceptions import UserError from decimal import Decimal from datetime import time, date, datetime from unittest.mock import MagicMock @@ -131,5 +132,54 @@ High 34,87 EUR 'rgxdate' ), date(2022, 3, 14)) + @with_transaction() + def test_waitlist_source_check_regex_validate(self): + """ create source, check validation of regex-code + """ + pool = Pool() + OSource = pool.get('investment.source') + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Date': nothing to repeat " + + r"at position 0 \[\*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check date', + 'rgxdate': '** multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert ** multiple repeat', + }]) + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Rate': multiple repeat " + + r"at position 6 \[rate \*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check rate', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate ** multiple repeat', + 'rgxident': 'identifiert -- multiple repeat', + }]) + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Identifier': multiple " + + r"repeat at position 13 \[identifiert \*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check rgxident', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert ** multiple repeat', + }]) + + OSource.create([{ + 'name': 'Check rgxident', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert -- multiple repeat', + }]) # end SourceTestCase diff --git a/update_wiz.py b/update_wiz.py index 64f9359..50a36a3 100644 --- a/update_wiz.py +++ b/update_wiz.py @@ -29,7 +29,7 @@ class UpdateSoureWizard(Wizard): if OnlineSource.update_rate(asset): to_run_activities.append(asset) - if len(to_run_activities) > 0: + if to_run_activities: Asset.after_update_actions(to_run_activities) return 'end' diff --git a/view/asset_list.xml b/view/asset_list.xml index e5c15bc..7837d85 100644 --- a/view/asset_list.xml +++ b/view/asset_list.xml @@ -4,12 +4,12 @@ The COPYRIGHT file at the top level of this repository contains the full copyright notices and license terms. --> - - - - - - - - + + + + + + + +