From 6789e6387298570eccd57dca67881636160e5a6c Mon Sep 17 00:00:00 2001 From: Frederik Jaeckel Date: Fri, 13 Jan 2023 13:08:52 +0100 Subject: [PATCH] OnlineSource: Query method prepared for external extension --- locale/de.po | 16 +++++++ locale/en.po | 16 +++++++ message.xml | 3 ++ onlinesource.py | 107 ++++++++++++++++++++++++++++++++----------- view/source_form.xml | 11 +++-- view/source_list.xml | 1 + 6 files changed, 124 insertions(+), 30 deletions(-) diff --git a/locale/de.po b/locale/de.po index fef6097..01e0604 100644 --- a/locale/de.po +++ b/locale/de.po @@ -42,6 +42,10 @@ msgctxt "model:ir.message,text:msg_err_unknown_content" msgid "failed to identify row content: %(linetxt)s" msgstr "Zeileninhalt konnte nicht identifiziert werden: %(linetxt)s" +msgctxt "model:ir.message,text:msg_querytype_web" +msgid "Extract from web page" +msgstr "aus Webseite auslesen" + ############## # ir.ui.menu # @@ -386,6 +390,10 @@ msgctxt "view:investment.source:" msgid "Purely javascript-based websites do not work here." msgstr "Rein javascriptbasierte Webseiten funktionieren hier nicht." +msgctxt "view:investment.source:" +msgid "Method: Extract from web page" +msgstr "Methode: aus Webseite auslesen" + msgctxt "field:investment.source,name:" msgid "Name" msgstr "Name" @@ -514,6 +522,14 @@ msgctxt "help:investment.source,fndident:" msgid "Identifier found during test query." msgstr "Bei der Testabfrage gefundener Bezeichner." +msgctxt "field:investment.source,query_method:" +msgid "Method" +msgstr "Methode" + +msgctxt "help:investment.source,query_method:" +msgid "Select the method to retrieve the data." +msgstr "Wählen Sie die Methode zum Abruf der Daten." + ################### # investment.rate # diff --git a/locale/en.po b/locale/en.po index e11dbea..fda56d9 100644 --- a/locale/en.po +++ b/locale/en.po @@ -30,6 +30,10 @@ msgctxt "model:ir.message,text:msg_err_unknown_content" msgid "failed to identify row content: %(linetxt)s" msgstr "failed to identify row content: %(linetxt)s" +msgctxt "model:ir.message,text:msg_querytype_web" +msgid "Extract from web page" +msgstr "Extract from web page" + msgctxt "model:ir.ui.menu,name:menu_investment" msgid "Investment" msgstr "Investment" @@ -338,6 +342,10 @@ msgctxt "view:investment.source:" msgid "Purely javascript-based websites do not work here." msgstr "Purely javascript-based websites do not work here." +msgctxt "view:investment.source:" +msgid "Method: Extract from web page" +msgstr "Method: Extract from web page" + msgctxt "field:investment.source,name:" msgid "Name" msgstr "Name" @@ -466,6 +474,14 @@ msgctxt "help:investment.source,fndident:" msgid "Identifier found during test query." msgstr "Identifier found during test query." +msgctxt "field:investment.source,query_method:" +msgid "Method" +msgstr "Method" + +msgctxt "help:investment.source,query_method:" +msgid "Select the method to retrieve the data." +msgstr "Select the method to retrieve the data." + msgctxt "model:investment.rate,name:" msgid "Rate" msgstr "Rate" diff --git a/message.xml b/message.xml index 3d5c831..dae31b6 100644 --- a/message.xml +++ b/message.xml @@ -17,6 +17,9 @@ full copyright notices and license terms. --> failed to identify row content: %(linetxt)s + + Extract from web page + diff --git a/onlinesource.py b/onlinesource.py index 65380db..076820e 100644 --- a/onlinesource.py +++ b/onlinesource.py @@ -11,6 +11,7 @@ from trytond.model import ModelView, ModelSQL, fields, Unique, Check from trytond.transaction import Transaction from trytond.pool import Pool from trytond.pyson import Eval, Bool +from trytond.i18n import gettext logger = logging.getLogger(__name__) @@ -39,34 +40,54 @@ fields_check = ['url', 'nsin', 'isin', 'symbol', 'text', 'http_state', \ 'fnddate', 'fndrate', 'fndident'] +STATES_WEB = { + 'invisible': Eval('query_method', '') != 'web', + 'required': Eval('query_method', '') == 'web', + } +DEPENDS_WEB = ['query_method'] + + class OnlineSource(ModelSQL, ModelView): 'Online Source' __name__ = 'investment.source' name = fields.Char(string='Name', required=True) - url = fields.Char(string='URL', required=True) + query_method = fields.Selection(string='Method', required=True, + help='Select the method to retrieve the data.', + selection='get_query_methods') + url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB) nohtml = fields.Boolean(string='Remove HTML', - help='Removes HTML tags before the text is interpreted.') - rgxdate = fields.Char(string='Date', required=True, - help='Regex code to find the date in the downloaded HTML file.') - rgxdatefmt = fields.Selection(string='Date format', required=True, - selection=sel_rgxdatefmt) - rgxrate = fields.Char(string='Rate', required=True, - help='Regex code to find the rate in the downloaded HTML file.') - rgxdecimal = fields.Selection(string='Decimal Separator', required=True, + help='Removes HTML tags before the text is interpreted.', + states={ + 'invisible': STATES_WEB['invisible'], + }, depends=DEPENDS_WEB) + rgxdate = fields.Char(string='Date', + help='Regex code to find the date in the downloaded HTML file.', + states=STATES_WEB, depends=DEPENDS_WEB) + rgxdatefmt = fields.Selection(string='Date format', selection=sel_rgxdatefmt, + states=STATES_WEB, depends=DEPENDS_WEB) + rgxrate = fields.Char(string='Rate', + help='Regex code to find the rate in the downloaded HTML file.', + states=STATES_WEB, depends=DEPENDS_WEB) + rgxdecimal = fields.Selection(string='Decimal Separator', help='Decimal separator for converting the market value into a number.', - selection=sel_rgxdecimal) + selection=sel_rgxdecimal, states=STATES_WEB, depends=DEPENDS_WEB) rgxident = fields.Char(string='Identifier', - help='Regex code to find the identifier in the downloaded HTML file.') + help='Regex code to find the identifier in the downloaded HTML file.', + states={ + 'invisible': STATES_WEB['invisible'], + }, depends=DEPENDS_WEB) rgxidtype = fields.Selection(string='ID-Type', selection=sel_rgxidtype, help='Type of identifier used to validate the result.', states={ 'required': Bool(Eval('rgxident', '')), - }, depends=['rgxident']) + 'invisible': STATES_WEB['invisible'], + }, depends=DEPENDS_WEB+['rgxident']) # field to test requests used_url = fields.Function(fields.Char(string='Used URL', readonly=True, - help='This URL is used to retrieve the HTML file.'), + help='This URL is used to retrieve the HTML file.', + states={'invisible': STATES_WEB['invisible']}, depends=DEPENDS_WEB), 'on_change_with_used_url') nsin = fields.Function(fields.Char(string='NSIN'), 'on_change_with_nsin', setter='set_test_value') @@ -93,6 +114,12 @@ class OnlineSource(ModelSQL, ModelView): super(OnlineSource, cls).__setup__() cls._order.insert(0, ('name', 'DESC')) + @classmethod + def default_query_method(cls): + """ default: web + """ + return 'web' + @classmethod def default_url(cls): """ defaul-url @@ -188,30 +215,56 @@ class OnlineSource(ModelSQL, ModelView): symbol = self.symbol, ) + @classmethod + def get_query_methods(cls): + """ get list of query-methods + """ + return [ + ('web', gettext('investment.msg_querytype_web')), + ] + @classmethod def set_test_value(cls, record, name, value): """ dont store it """ pass + @classmethod + def run_query_method(cls, osource, isin, nsin, symbol, debug=False): + """ run selected query to retrive data + result: { + 'text': raw-text from query - for debug, + 'http_state': state of query, + 'date': date() if success, + 'rate': Decimal() if success, + 'code': identifier - isin/nsin/symbol + } + """ + OSourc = Pool().get('investment.source') + + if getattr(osource, 'query_method', None) == 'web': + return OSourc.read_from_website( + osource, + isin = isin, + nsin = nsin, + symbol = symbol, + debug = debug, + ) + def call_online_source(self): """ use updated values to call online-source, for testing parameters """ OSourc = Pool().get('investment.source') - result = OSourc.read_from_website( - self, - isin = self.isin, - nsin = self.nsin, - symbol = self.symbol, - debug = True, - ) - self.text = result.get('text', None) - self.http_state = result.get('http_state', None) - self.fnddate = result.get('date', None) - self.fndrate = result.get('rate', None) - self.fndident = result.get('code', None) + result = OSourc.run_query_method(self, self.isin, self.nsin, + self.symbol, debug=True) + if result is not None: + self.text = result.get('text', None) + self.http_state = result.get('http_state', None) + self.fnddate = result.get('date', None) + self.fndrate = result.get('rate', None) + self.fndident = result.get('code', None) def get_url_with_parameter(self, isin=None, nsin=None, symbol=None): """ generate url @@ -235,12 +288,12 @@ class OnlineSource(ModelSQL, ModelView): return for updtsource in asset.updtsources: - rate_data = cls.read_from_website( + rate_data = cls.run_query_method( updtsource, isin = asset.isin, nsin = asset.wkn, symbol = asset.secsymb, - ) + ) if len(updtsource.rgxident or '') > 0: # check result - same code? diff --git a/view/source_form.xml b/view/source_form.xml index eb03503..b4ea9ff 100644 --- a/view/source_form.xml +++ b/view/source_form.xml @@ -5,16 +5,21 @@ full copyright notices and license terms. -->