From 3b9de6c0bb6764344b729ee37c2e981d12759a26 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Thu, 22 Jun 2023 17:28:58 +0200 Subject: [PATCH 01/12] asset: fixed possible exceptions --- asset.py | 65 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/asset.py b/asset.py index d599d7b..06bade6 100644 --- a/asset.py +++ b/asset.py @@ -330,23 +330,24 @@ class Asset(SymbolMixin, ModelSQL, ModelView): """ cursor = Transaction().connection.cursor() - (query, tab_asset) = cls.get_name_symbol_sql() - query.where = tab_asset.id.in_([x.id for x in assets]) - - cursor.execute(*query) - records = cursor.fetchall() result = {x: {y.id: None for y in assets} for x in names} - for record in records: - values = { - 'name': record[1], - 'product_uom': record[2], - 'symbol': record[3], - 'asset_symbol': record[0], - } + (query, tab_asset) = cls.get_name_symbol_sql() + if assets: + query.where = tab_asset.id.in_([x.id for x in assets]) + cursor.execute(*query) + records = cursor.fetchall() - for name in names: - result[name][record[0]] = values[name] + for record in records: + values = { + 'name': record[1], + 'product_uom': record[2], + 'symbol': record[3], + 'asset_symbol': record[0], + } + + for name in names: + result[name][record[0]] = values[name] return result @classmethod @@ -384,32 +385,32 @@ class Asset(SymbolMixin, ModelSQL, ModelView): def get_rate_data(cls, assets, names): """ get date and rate of asset """ - Asset2 = Pool().get('investment.asset') - cursor = Transaction().connection.cursor() - (query, tab_asset) = cls.get_rate_data_sql() - query.where = tab_asset.id.in_([x.id for x in assets]) - - cursor.execute(*query) - records = cursor.fetchall() result = {x: {y.id: None for y in assets} for x in names} - for record in records: - (id1, rate1, date1, id_rate) = record + if assets: + (query, tab_asset) = cls.get_rate_data_sql() + query.where = tab_asset.id.in_([x.id for x in assets]) + curr_digits = {x.id: x.currency_digits for x in assets} - asset = Asset2(id1) - exp = Decimal(Decimal(1) / 10 ** (asset.currency_digits or 4)) + cursor.execute(*query) + records = cursor.fetchall() - values = { - 'rate': record[1].quantize(exp), - 'date': record[2], - 'change_symbol': id_rate, - } + for record in records: + (id1, rate1, date1, id_rate) = record - for name in names: - result[name][record[0]] = values[name] + curr_dig = curr_digits.get(id1, 4) + exp = Decimal(Decimal(1) / 10 ** curr_dig) + values = { + 'rate': record[1].quantize(exp), + 'date': record[2], + 'change_symbol': id_rate, + } + + for name in names: + result[name][record[0]] = values[name] return result @classmethod From 38e2c34a53f9bf6571eb3941194e4fc164485cdc Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 23 Jun 2023 16:02:31 +0200 Subject: [PATCH 02/12] asset/rate: speed up percent-queries --- asset.py | 48 ++++++++++++++++++++++-------------------------- rate.py | 7 +++++++ 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/asset.py b/asset.py index 06bade6..9d4ca63 100644 --- a/asset.py +++ b/asset.py @@ -473,8 +473,6 @@ class Asset(SymbolMixin, ModelSQL, ModelView): """ pool = Pool() Rate = pool.get('investment.rate') - Asset2 = pool.get('investment.asset') - tab_asset = Asset2.__table__() tab_rate1 = Rate.__table__() tab_rate2 = Rate.__table__() context = Transaction().context @@ -482,17 +480,14 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query_date = context.get('qdate', CurrentDate()) where_asset = tab_rate1.date <= query_date if isinstance(asset_ids, list): - where_asset &= tab_asset.id.in_(asset_ids) + where_asset &= tab_rate1.asset.in_(asset_ids) - tab_today = tab_asset.join( - tab_rate1, - condition=tab_asset.id == tab_rate1.asset, - ).select( - tab_asset.id, + tab_today = tab_rate1.select( + tab_rate1.asset.as_('id'), tab_rate1.date, tab_rate1.rate, - distinct_on=[tab_asset.id], - order_by=[tab_asset.id, tab_rate1.date.desc], + distinct_on=[tab_rate1.asset], + order_by=[tab_rate1.asset, tab_rate1.date.desc], where=where_asset, ) @@ -613,23 +608,24 @@ class Asset(SymbolMixin, ModelSQL, ModelView): exp = Decimal(Decimal(1) / 10 ** digits_percent) asset_id_lst = [x.id for x in assets] - for x in names: - tab_percent = cls.get_percentage_sql( - days={ - 'change_day1': 0, - 'change_month1': 30, - 'change_month3': 90, - 'change_month6': 180, - 'change_month12': 365, - }[x], - asset_ids=asset_id_lst, - ) - cursor.execute(*tab_percent) - records = cursor.fetchall() + if asset_id_lst and names: + for x in names: + tab_percent = cls.get_percentage_sql( + days={ + 'change_day1': 0, + 'change_month1': 30, + 'change_month3': 90, + 'change_month6': 180, + 'change_month12': 365, + }[x], + asset_ids=asset_id_lst, + ) + cursor.execute(*tab_percent) + records = cursor.fetchall() - for record in records: - result[x][record[0]] = record[3].quantize(exp) \ - if record[3] is not None else None + for record in records: + result[x][record[0]] = record[3].quantize(exp) \ + if record[3] is not None else None return result @classmethod diff --git a/rate.py b/rate.py index eef697c..6379e6b 100644 --- a/rate.py +++ b/rate.py @@ -53,6 +53,13 @@ class Rate(SymbolMixin, ModelSQL, ModelView): Index( t, (t.rate, Index.Range())), + Index( + t, + (t.asset, Index.Equality())), + Index( + t, + (t.asset, Index.Equality()), + (t.date, Index.Range(order='DESC'))), }) @classmethod From 6e77058946779899198f92c477b75fd3e8c773f3 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 23 Jun 2023 16:29:10 +0200 Subject: [PATCH 03/12] asset: avoid exceptions in get_identifiers() --- asset.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/asset.py b/asset.py index 9d4ca63..98286af 100644 --- a/asset.py +++ b/asset.py @@ -853,20 +853,19 @@ class Asset(SymbolMixin, ModelSQL, ModelView): cursor = Transaction().connection.cursor() result = {x: {y.id: None for y in assets} for x in names} + if assets: + query = cls.get_identifier_sql(tab_asset) + query.where = tab_asset.id.in_([x.id for x in assets]) - query = cls.get_identifier_sql(tab_asset) - query.where = tab_asset.id.in_([x.id for x in assets]) + cursor.execute(*query) + l1 = cursor.fetchall() - cursor.execute(*query) - l1 = cursor.fetchall() - - for x in l1: - (id1, wkn, secsymb, isin) = x - r1 = {'wkn': wkn, 'secsymb': secsymb, 'isin': isin} - - for n in names: - result[n][id1] = r1[n] + for x in l1: + (id1, wkn, secsymb, isin) = x + r1 = {'wkn': wkn, 'secsymb': secsymb, 'isin': isin} + for n in names: + result[n][id1] = r1[n] return result def get_rec_name(self, name): From ff5893b4510ad7a656cb126e6360d094e4f7b249 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 23 Jun 2023 16:41:39 +0200 Subject: [PATCH 04/12] diagram: simplify queries --- diagram.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/diagram.py b/diagram.py index 514a2ad..08c7cab 100644 --- a/diagram.py +++ b/diagram.py @@ -60,16 +60,16 @@ class GraphDef(metaclass=PoolMeta): return None if self.scaling == 'alldata': - query = [('asset.id', '=', self.asset.id)] + query = [('asset', '=', self.asset.id)] elif self.scaling == 'view': query = [ - ('asset.id', '=', self.asset.id), + ('asset', '=', self.asset.id), ('date', '>=', self.chart.used_start_date()), ('date', '<=', self.chart.used_end_date()), ] elif self.scaling == 'six': query = [ - ('asset.id', '=', self.asset.id), + ('asset', '=', self.asset.id), ('date', '>=', self.chart.used_start_date() - timedelta(days=180)), ('date', '<=', self.chart.used_end_date()), @@ -104,12 +104,12 @@ class ChartPoint(metaclass=PoolMeta): before = Rate.search([ ('date', '<', query_date), - ('asset.id', '=', asset_id), + ('asset', '=', asset_id), ], limit=1, order=[('date', 'DESC')]) after = Rate.search([ ('date', '>', query_date), - ('asset.id', '=', asset_id), + ('asset', '=', asset_id), ], limit=1, order=[('date', 'ASC')]) if (len(before) == 1) and (len(after) == 1): From aed948fb8aeb57a2aa2e1e9417bddaa32e6c3048 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 23 Jun 2023 21:40:55 +0200 Subject: [PATCH 05/12] asset-list: remove percent-symbol to speed up list view --- view/asset_list.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/view/asset_list.xml b/view/asset_list.xml index daee4c5..e5c15bc 100644 --- a/view/asset_list.xml +++ b/view/asset_list.xml @@ -4,10 +4,10 @@ The COPYRIGHT file at the top level of this repository contains the full copyright notices and license terms. --> - - - - + + + + From faefea3b5f265f9c2e78df83c3659f7dd79c054c Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 1 Dec 2023 13:29:46 +0100 Subject: [PATCH 06/12] formatting --- asset.py | 9 +++++---- const.py | 8 ++++++++ import_wiz.py | 3 ++- onlinesource.py | 3 ++- 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 const.py diff --git a/asset.py b/asset.py index 98286af..dd62caa 100644 --- a/asset.py +++ b/asset.py @@ -15,6 +15,7 @@ from sql.functions import CurrentDate, CurrentTimestamp, Round, Extract from sql.conditionals import Case, Coalesce, NullIf from sql import Literal from .diagram import Concat2 +from .const import DEF_NONE digits_percent = 2 @@ -100,8 +101,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView): string='URL', help='URL for data retrieval.', states={ - 'invisible': Eval('updturl_enable', False) == False, - 'required': Eval('updturl_enable', False) == True, + 'invisible': ~Eval('updturl_enable', False), + 'required': Eval('updturl_enable', False), }, depends=['updturl_enable']) updturl_enable = fields.Function(fields.Boolean( string='URL required', readonly=True, @@ -191,7 +192,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.id, tab_asset.updtsource, - where=tab_asset.updtsource != None, + where=tab_asset.updtsource != DEF_NONE, ) cursor.execute(*query) records = cursor.fetchall() @@ -838,7 +839,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): ).select( tab_asset.id, where=Operator(field_qu, clause[2]) & - (field_qu != None), + (field_qu != DEF_NONE), ) return [('id', 'in', query)] diff --git a/const.py b/const.py new file mode 100644 index 0000000..a01e781 --- /dev/null +++ b/const.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# This file is part of the cashbook-module from m-ds.de for Tryton. +# The COPYRIGHT file at the top level of this repository contains the +# full copyright notices and license terms. + + +DEF_TRUE = True +DEF_NONE = None diff --git a/import_wiz.py b/import_wiz.py index 72e7d4e..4722557 100644 --- a/import_wiz.py +++ b/import_wiz.py @@ -173,7 +173,8 @@ class ImportWizard(Wizard): datefmt='dd%sdd' % dec_divider, colnr='2')) - if isinstance(date_val, date) and isinstance(rate_val, Decimal): + if isinstance(date_val, date) and isinstance( + rate_val, Decimal): result.append({'date': date_val, 'rate': rate_val}) # date range diff --git a/onlinesource.py b/onlinesource.py index c881aa5..a74696a 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -86,7 +86,8 @@ class OnlineSource(ModelSQL, ModelView): states=STATES_WEB, depends=DEPENDS_WEB) rgxdecimal = fields.Selection( string='Decimal Separator', - help='Decimal separator for converting the market value into a number.', + help='Decimal separator for converting the market ' + + 'value into a number.', selection=sel_rgxdecimal, states=STATES_WEB, depends=DEPENDS_WEB) rgxident = fields.Char( string='Identifier', From 18ef1aaeb6a68b0d85072b4969e51018a6d5be76 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 1 Dec 2023 13:31:43 +0100 Subject: [PATCH 07/12] Tryton 7.0 --- README.rst | 6 +++--- setup.py | 4 ++-- tryton.cfg | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 1f48f27..db3f6ab 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ pip install mds-investment Requires ======== -- Tryton 6.8 +- Tryton 7.0 How to ====== @@ -22,6 +22,6 @@ You can define the course sources yourself. Changes ======= -*6.8.24 - 07.06.2023* +*7.0.0 - 01.12.2023* -- portet to Tryton 6.8 +- compatibility to Tryton 7.0 diff --git a/setup.py b/setup.py index 58979c0..29f2a96 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,8 @@ with open(path.join(here, 'versiondep.txt'), encoding='utf-8') as f: modversion[l2[0]] = {'min': l2[1], 'max': l2[2], 'prefix': l2[3]} # tryton-version -major_version = 6 -minor_version = 8 +major_version = 7 +minor_version = 0 requires = ['requests>=2.26', 'html2text'] for dep in info.get('depends', []): diff --git a/tryton.cfg b/tryton.cfg index 4ba63f6..39f64d0 100644 --- a/tryton.cfg +++ b/tryton.cfg @@ -1,5 +1,5 @@ [tryton] -version=6.8.24 +version=7.0.0 depends: ir res From c2df3886925e213a7d263ee786b6eac8036ef3dc Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sun, 3 Dec 2023 18:20:22 +0100 Subject: [PATCH 08/12] asset: columns optional --- view/asset_list.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/view/asset_list.xml b/view/asset_list.xml index e5c15bc..7837d85 100644 --- a/view/asset_list.xml +++ b/view/asset_list.xml @@ -4,12 +4,12 @@ The COPYRIGHT file at the top level of this repository contains the full copyright notices and license terms. --> - - - - - - - - + + + + + + + + From 9f74b8fbf7e43c09683992439dcab6f4e7a69564 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sun, 21 Jan 2024 18:55:21 +0100 Subject: [PATCH 09/12] check regex-code when save online-source --- locale/de.po | 4 ++++ locale/en.po | 16 ++++++++++++++++ message.xml | 3 +++ onlinesource.py | 24 +++++++++++++++++++++--- tests/source.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 3 deletions(-) diff --git a/locale/de.po b/locale/de.po index 5d55abe..2e027be 100644 --- a/locale/de.po +++ b/locale/de.po @@ -50,6 +50,10 @@ msgctxt "model:ir.message,text:msg_missing_url" msgid "URL for the online source '%(oname)s' is missing." msgstr "URL für die Onlinequelle '%(oname)s' fehlt." +msgctxt "model:ir.message,text:msg_bug_in_regexquery" +msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" +msgstr "Fehler in Regex-Code des Feldes '%(fname)s': %(errmsg)s [%(code)s]" + ############## # ir.ui.menu # diff --git a/locale/en.po b/locale/en.po index 7e0923e..7d76175 100644 --- a/locale/en.po +++ b/locale/en.po @@ -38,6 +38,10 @@ msgctxt "model:ir.message,text:msg_missing_url" msgid "URL for the online source '%(oname)s' is missing." msgstr "URL for the online source '%(oname)s' is missing." +msgctxt "model:ir.message,text:msg_bug_in_regexquery" +msgid "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" +msgstr "Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s]" + msgctxt "model:ir.ui.menu,name:menu_investment" msgid "Investment" msgstr "Investment" @@ -294,6 +298,18 @@ msgctxt "selection:investment.asset,updtdays:" msgid "Mon - Sun" msgstr "Mon - Sun" +msgctxt "field:investment.asset,updturl:" +msgid "URL" +msgstr "URL" + +msgctxt "help:investment.asset,updturl:" +msgid "URL for data retrieval." +msgstr "URL for data retrieval." + +msgctxt "field:investment.asset,updturl_enable:" +msgid "URL required" +msgstr "URL required" + msgctxt "model:investment.asset_source_rel,name:" msgid "Asset Source Relation" msgstr "Asset Source Relation" diff --git a/message.xml b/message.xml index 2ef9571..2d681ef 100644 --- a/message.xml +++ b/message.xml @@ -23,6 +23,9 @@ full copyright notices and license terms. --> URL for the online source '%(oname)s' is missing. + + Error in regex code of field '%(fname)s': %(errmsg)s [%(code)s] + diff --git a/onlinesource.py b/onlinesource.py index a74696a..4060ae4 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -260,6 +260,15 @@ class OnlineSource(ModelSQL, ModelView): """ pass + @classmethod + def validate(cls, records): + """ check regex-code + """ + for record in records: + for x in ['rgxdate', 'rgxrate', 'rgxident']: + if x: + record.get_regex_result('', x) + @classmethod def run_query_method(cls, osource, isin, nsin, symbol, url, debug=False): """ run selected query to retrive data @@ -380,13 +389,22 @@ class OnlineSource(ModelSQL, ModelView): def get_regex_result(self, html_text, field_name): """ run regex on html-text, convert result """ + OSource = Pool().get('investment.source') + rgxcode = getattr(self, field_name) or '' if len(rgxcode) == 0: return None - search_result = re.compile(rgxcode).search(html_text) - if search_result is None: - return None + try: + search_result = re.compile(rgxcode).search(html_text) + if search_result is None: + return None + except Exception as e1: + raise UserError(gettext( + 'investment.msg_bug_in_regexquery', + errmsg=str(e1), + fname=getattr(OSource, field_name).string, + code=rgxcode)) try: result = search_result.group(1) diff --git a/tests/source.py b/tests/source.py index f784e4c..43ac22b 100644 --- a/tests/source.py +++ b/tests/source.py @@ -6,6 +6,7 @@ from trytond.tests.test_tryton import with_transaction from trytond.pool import Pool from trytond.transaction import Transaction +from trytond.exceptions import UserError from decimal import Decimal from datetime import time, date, datetime from unittest.mock import MagicMock @@ -131,5 +132,54 @@ High 34,87 EUR 'rgxdate' ), date(2022, 3, 14)) + @with_transaction() + def test_waitlist_source_check_regex_validate(self): + """ create source, check validation of regex-code + """ + pool = Pool() + OSource = pool.get('investment.source') + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Date': nothing to repeat " + + r"at position 0 \[\*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check date', + 'rgxdate': '** multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert ** multiple repeat', + }]) + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Rate': multiple repeat " + + r"at position 6 \[rate \*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check rate', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate ** multiple repeat', + 'rgxident': 'identifiert -- multiple repeat', + }]) + + self.assertRaisesRegex( + UserError, + r"Error in regex code of field 'Identifier': multiple " + + r"repeat at position 13 \[identifiert \*+ multiple repeat\]", + OSource.create, + [{ + 'name': 'Check rgxident', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert ** multiple repeat', + }]) + + OSource.create([{ + 'name': 'Check rgxident', + 'rgxdate': '-- multiple repeat', + 'rgxrate': 'rate -- multiple repeat', + 'rgxident': 'identifiert -- multiple repeat', + }]) # end SourceTestCase From 4a2135512fe4938cf9897e98cd669efe2147697b Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sun, 21 Jan 2024 19:08:41 +0100 Subject: [PATCH 10/12] accept http 410 on server call --- onlinesource.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/onlinesource.py b/onlinesource.py index 4060ae4..4ebad01 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -313,19 +313,17 @@ class OnlineSource(ModelSQL, ModelView): """ generate url """ if self.fixed_url is True: - if url is None: + if not url: raise UserError(gettext( 'investment.msg_missing_url', - oname=self.rec_name, - )) + oname=self.rec_name)) return url - else: - if self.url: - return Template(self.url).substitute({ - 'isin': isin if isin is not None else '', - 'nsin': nsin if nsin is not None else '', - 'symbol': symbol if symbol is not None else '', - }) + + if self.url: + return Template(self.url).substitute({ + 'isin': isin if isin is not None else '', + 'nsin': nsin if nsin is not None else '', + 'symbol': symbol if symbol is not None else ''}) @classmethod def update_rate(cls, asset): @@ -455,7 +453,7 @@ class OnlineSource(ModelSQL, ModelView): 'msg': res1.reason, } - if res1.status_code in [200, 204]: + if res1.status_code in [200, 204, 410]: html = res1.text # remove html-tags From ab6ab32d0a78b730e59223f65b570b43b50e5e40 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sun, 21 Jan 2024 19:14:05 +0100 Subject: [PATCH 11/12] update online source 'www.sbroker.de' --- sources_def.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sources_def.xml b/sources_def.xml index 1f49541..ff5959a 100644 --- a/sources_def.xml +++ b/sources_def.xml @@ -71,9 +71,9 @@ full copyright notices and license terms. --> www.sbroker.de https://www.sbroker.de/sbl/mdaten_analyse/dksuche_a?SEARCH_VALUE=${isin} - \nDatum / Uhrzeit: (\d+\.\d+\.\d+) / \d+:\d+\s+\n + \nDatum\s*(?:\/ Uhrzeit)*: (\d+\.\d+\.\d+)\s*(?:\/ \d+:\d+)*\s*\n %d.%m.%y - Kurs aktuell .* (\d+,\d+)\s+EUR.*\n + (?:Kurs aktuell|Rucknahmepreis)\s+[()\w\./\[\]!]+\s+(\d+,\d+)\s+(EUR|€) , \nWKN / ISIN: [A-Z,0-9]+ / ([A-Z,0-9]+)\s+\n isin From 4a3d9966002de4557ed510f1c90d8d211b25ca25 Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Sun, 21 Jan 2024 19:37:48 +0100 Subject: [PATCH 12/12] optimize code --- asset.py | 94 +++++++++++++++++-------------------------------- diagram.py | 2 +- import_wiz.py | 12 +++---- onlinesource.py | 39 ++++++++------------ update_wiz.py | 2 +- 5 files changed, 55 insertions(+), 94 deletions(-) diff --git a/asset.py b/asset.py index dd62caa..578de36 100644 --- a/asset.py +++ b/asset.py @@ -32,7 +32,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): name = fields.Function(fields.Char( string='Name', readonly=True), - 'get_name_symbol', searcher='search_rec_name') + 'get_name_symbol', searcher='search_rec_name') company = fields.Many2One( string='Company', model_name='company.company', required=True, ondelete="RESTRICT") @@ -46,12 +46,9 @@ class Asset(SymbolMixin, ModelSQL, ModelView): uom = fields.Many2One( string='UOM', required=True, model_name='product.uom', ondelete='RESTRICT', - states={ - 'readonly': ~Bool(Eval('product')), - }, - domain=[ - ('category', '=', Eval('product_uom')), - ], depends=['product_uom', 'product']) + states={'readonly': ~Bool(Eval('product'))}, + domain=[('category', '=', Eval('product_uom'))], + depends=['product_uom', 'product']) symbol = fields.Function(fields.Char( string='UOM', readonly=True), 'get_name_symbol', searcher='search_uom_symbol') @@ -74,9 +71,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): model_name='currency.currency', ondelete='RESTRICT') currency_digits = fields.Integer( string='Digits', required=True, - domain=[ - ('currency_digits', '>=', 0), - ('currency_digits', '<=', 6)]) + domain=[('currency_digits', '>=', 0), ('currency_digits', '<=', 6)]) wkn = fields.Function(fields.Char( string='NSIN', readonly=True, @@ -112,9 +107,8 @@ class Asset(SymbolMixin, ModelSQL, ModelView): string='Select days', required=True, selection=sel_updtdays) updttime = fields.Time( string='Time', - states={ - 'readonly': ~Bool(Eval('updtsources')), - }, depends=['updtsources']) + states={'readonly': ~Bool(Eval('updtsources'))}, + depends=['updtsources']) nextupdate = fields.Function(fields.DateTime( string='Next Update', readonly=True), 'get_nextupdates', searcher='search_nextupdate') @@ -201,7 +195,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): 'source': x[1], } for x in records] - if len(to_create) > 0: + if to_create: AssetSourceRel.create(to_create) asset_table.drop_column('updtsource') @@ -262,7 +256,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): def on_change_updtsources(self): """ clear time-fields """ - if len(self.updtsources) == 0: + if not self.updtsources: self.updttime = None else: self.updttime = time(11, 30) @@ -423,8 +417,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.id, - where=Operator(tab_query.date, clause[2]), - ) + where=Operator(tab_query.date, clause[2])) return [('id', 'in', query)] @classmethod @@ -436,8 +429,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.id, - where=Operator(tab_query.rate, clause[2]), - ) + where=Operator(tab_query.rate, clause[2])) return [('id', 'in', query)] @staticmethod @@ -449,8 +441,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.date, - where=tab_query.id == table.id, - ) + where=tab_query.id == table.id) return [query] @staticmethod @@ -462,8 +453,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_query.select( tab_query.rate, - where=tab_query.id == table.id, - ) + where=tab_query.id == table.id) return [query] @classmethod @@ -489,8 +479,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_rate1.rate, distinct_on=[tab_rate1.asset], order_by=[tab_rate1.asset, tab_rate1.date.desc], - where=where_asset, - ) + where=where_asset) days_diff = days + 5 query = tab_today.join( @@ -506,8 +495,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): (tab_today.rate * 100.0 / NullIf(tab_rate2.rate, 0.00) - 100.0).as_('percent'), distinct_on=[tab_today.id], - order_by=[tab_today.id, tab_rate2.date.desc] - ) + order_by=[tab_today.id, tab_rate2.date.desc]) return query @staticmethod @@ -520,8 +508,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -534,8 +521,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -548,8 +534,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -562,8 +547,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -576,8 +560,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_asset.select( tab_asset.percent, - where=tab_asset.id == table.id, - ) + where=tab_asset.id == table.id) return [query] @classmethod @@ -619,14 +602,13 @@ class Asset(SymbolMixin, ModelSQL, ModelView): 'change_month6': 180, 'change_month12': 365, }[x], - asset_ids=asset_id_lst, - ) + asset_ids=asset_id_lst) cursor.execute(*tab_percent) records = cursor.fetchall() for record in records: result[x][record[0]] = record[3].quantize(exp) \ - if record[3] is not None else None + if record[3] is not None else None return result @classmethod @@ -660,8 +642,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_asset.updtdays, tab_asset.updttime, distinct_on=[tab_asset.id], - order_by=[tab_asset.id, tab_rate.date.desc], - ) + order_by=[tab_asset.id, tab_rate.date.desc]) query = tab_date.select( tab_date.id, @@ -673,8 +654,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): (Extract('dow', tab_date.date) == 6), tab_date.date + Literal(2)), else_=tab_date.date, - ) + tab_date.updttime).as_('updttime'), - ) + ) + tab_date.updttime).as_('updttime')) return query @classmethod @@ -688,8 +668,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_updt.select( tab_updt.id, tab_updt.updttime, - where=tab_updt.id.in_([x.id for x in assets]), - ) + where=tab_updt.id.in_([x.id for x in assets])) cursor.execute(*query) records = cursor.fetchall() @@ -713,8 +692,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_updt.select( tab_updt.id, - where=Operator(tab_updt.updttime, clause[2]), - ) + where=Operator(tab_updt.updttime, clause[2])) return [('id', 'in', query)] @classmethod @@ -751,8 +729,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): tab_asset.id, tab_wkn.code.as_('wkn'), tab_secsymb.code.as_('secsymb'), - tab_isin.code.as_('isin'), - ) + tab_isin.code.as_('isin')) return query @staticmethod @@ -776,8 +753,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): condition=tab_templ.id == tab_prod.template ).select( tab_templ.name, - where=tab_asset.id == table.id - ) + where=tab_asset.id == table.id) return [query] @staticmethod @@ -790,8 +766,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'wkn'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @staticmethod @@ -804,8 +779,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'isin'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @staticmethod @@ -818,8 +792,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): query = tab_ids.select( getattr(tab_ids, 'secsymb'), - where=tab_ids.id == table.id, - ) + where=tab_ids.id == table.id) return [query] @classmethod @@ -839,8 +812,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): ).select( tab_asset.id, where=Operator(field_qu, clause[2]) & - (field_qu != DEF_NONE), - ) + (field_qu != DEF_NONE)) return [('id', 'in', query)] @@ -913,7 +885,7 @@ class Asset(SymbolMixin, ModelSQL, ModelView): if OnlineSource.update_rate(asset): to_run_activities.append(asset) - if len(to_run_activities) > 0: + if to_run_activities: cls.after_update_actions(to_run_activities) # end Asset diff --git a/diagram.py b/diagram.py index 08c7cab..1d654e9 100644 --- a/diagram.py +++ b/diagram.py @@ -95,7 +95,7 @@ class ChartPoint(metaclass=PoolMeta): """ Rate = Pool().get('investment.rate') - if keyname is None: + if not keyname: return None # check if query is for us diff --git a/import_wiz.py b/import_wiz.py index 4722557..5a4d097 100644 --- a/import_wiz.py +++ b/import_wiz.py @@ -84,14 +84,14 @@ class ImportWizard(Wizard): pool = Pool() ImportWiz = pool.get('investment.imp_wiz', type='wizard') - if self.start.file_ is not None: + if self.start.file_: (lines, max_date, min_date) = ImportWiz.read_csv_file( - self.start.file_.decode('utf8'), - dec_divider=self.start.dec_divider, - date_fmt=self.start.date_fmt, - delimiter=self.start.field_delimiter) + self.start.file_.decode('utf8'), + dec_divider=self.start.dec_divider, + date_fmt=self.start.date_fmt, + delimiter=self.start.field_delimiter) - if len(lines) > 0: + if lines: ImportWiz.upload_rates( self.start.asset, lines, min_date, max_date) diff --git a/onlinesource.py b/onlinesource.py index 4ebad01..abda8b4 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -63,16 +63,14 @@ class OnlineSource(ModelSQL, ModelView): url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB) fixed_url = fields.Boolean( string='Fixed URL', - states={ - 'invisible': Eval('query_method', '') != 'web', - }, depends=DEPENDS_WEB, + states={'invisible': Eval('query_method', '') != 'web'}, + depends=DEPENDS_WEB, help='URL must be defined at investment record.') nohtml = fields.Boolean( string='Remove HTML', help='Removes HTML tags before the text is interpreted.', - states={ - 'invisible': STATES_WEB['invisible'], - }, depends=DEPENDS_WEB) + states={'invisible': STATES_WEB['invisible']}, + depends=DEPENDS_WEB) rgxdate = fields.Char( string='Date', help='Regex code to find the date in the downloaded HTML file.', @@ -92,9 +90,8 @@ class OnlineSource(ModelSQL, ModelView): rgxident = fields.Char( string='Identifier', help='Regex code to find the identifier in the downloaded HTML file.', - states={ - 'invisible': STATES_WEB['invisible'], - }, depends=DEPENDS_WEB) + states={'invisible': STATES_WEB['invisible']}, + depends=DEPENDS_WEB) rgxidtype = fields.Selection( string='ID-Type', selection=sel_rgxidtype, help='Type of identifier used to validate the result.', @@ -243,16 +240,13 @@ class OnlineSource(ModelSQL, ModelView): isin=self.isin, nsin=self.nsin, symbol=self.symbol, - url=self.url, - ) + url=self.url) @classmethod def get_query_methods(cls): """ get list of query-methods """ - return [ - ('web', gettext('investment.msg_querytype_web')), - ] + return [('web', gettext('investment.msg_querytype_web'))] @classmethod def set_test_value(cls, record, name, value): @@ -289,8 +283,7 @@ class OnlineSource(ModelSQL, ModelView): nsin=nsin, symbol=symbol, debug=debug, - url=url, - ) + url=url) def call_online_source(self): """ use updated values to call online-source, @@ -301,7 +294,7 @@ class OnlineSource(ModelSQL, ModelView): result = OSourc.run_query_method( self, self.isin, self.nsin, self.url, self.symbol, debug=True) - if result is not None: + if result: self.text = result.get('text', None) self.http_state = result.get('http_state', None) self.fnddate = result.get('date', None) @@ -342,8 +335,7 @@ class OnlineSource(ModelSQL, ModelView): isin=asset.isin, nsin=asset.wkn, symbol=asset.secsymb, - url=asset.updturl, - ) + url=asset.updturl) if len(updtsource.rgxident or '') > 0: # check result - same code? @@ -360,15 +352,13 @@ class OnlineSource(ModelSQL, ModelView): 'update_rate: got wrong code ' + '"%(wrong)s" - expected "%(exp)s"' % { 'exp': asset_code, - 'wrong': code, - }) + 'wrong': code}) continue to_create = { 'date': rate_data.get('date', None), 'rate': rate_data.get('rate', None), - 'asset': asset.id, - } + 'asset': asset.id} if (to_create['date'] is not None) and \ (to_create['rate'] is not None): # check if exists @@ -443,8 +433,7 @@ class OnlineSource(ModelSQL, ModelView): isin=isin, nsin=nsin, symbol=symbol, - url=url, - ), + url=url), allow_redirects=True, timeout=5.0) diff --git a/update_wiz.py b/update_wiz.py index 64f9359..50a36a3 100644 --- a/update_wiz.py +++ b/update_wiz.py @@ -29,7 +29,7 @@ class UpdateSoureWizard(Wizard): if OnlineSource.update_rate(asset): to_run_activities.append(asset) - if len(to_run_activities) > 0: + if to_run_activities: Asset.after_update_actions(to_run_activities) return 'end'