mdsfred/investment
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OnlineSource: Query method prepared for external extension

Browse source
This commit is contained in:
Frederik Jaeckel 2023-01-13 13:08:52 +01:00
parent 4bb7641959
commit 73dac7b33c
6 changed files with 124 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
locale/de.po
View file

@ -42,6 +42,10 @@ msgctxt "model:ir.message,text:msg_err_unknown_content"
msgid "failed to identify row content: %(linetxt)s"
msgstr "Zeileninhalt konnte nicht identifiziert werden: %(linetxt)s"
msgctxt "model:ir.message,text:msg_querytype_web"
msgid "Extract from web page"
msgstr "aus Webseite auslesen"
##############
# ir.ui.menu #
@ -386,6 +390,10 @@ msgctxt "view:investment.source:"
msgid "Purely javascript-based websites do not work here."
msgstr "Rein javascriptbasierte Webseiten funktionieren hier nicht."
msgctxt "view:investment.source:"
msgid "Method: Extract from web page"
msgstr "Methode: aus Webseite auslesen"
msgctxt "field:investment.source,name:"
msgid "Name"
msgstr "Name"
@ -514,6 +522,14 @@ msgctxt "help:investment.source,fndident:"
msgid "Identifier found during test query."
msgstr "Bei der Testabfrage gefundener Bezeichner."
msgctxt "field:investment.source,query_method:"
msgid "Method"
msgstr "Methode"
msgctxt "help:investment.source,query_method:"
msgid "Select the method to retrieve the data."
msgstr "Wählen Sie die Methode zum Abruf der Daten."
###################
# investment.rate #

16
locale/en.po
View file

@ -30,6 +30,10 @@ msgctxt "model:ir.message,text:msg_err_unknown_content"
msgid "failed to identify row content: %(linetxt)s"
msgstr "failed to identify row content: %(linetxt)s"
msgctxt "model:ir.message,text:msg_querytype_web"
msgid "Extract from web page"
msgstr "Extract from web page"
msgctxt "model:ir.ui.menu,name:menu_investment"
msgid "Investment"
msgstr "Investment"
@ -338,6 +342,10 @@ msgctxt "view:investment.source:"
msgid "Purely javascript-based websites do not work here."
msgstr "Purely javascript-based websites do not work here."
msgctxt "view:investment.source:"
msgid "Method: Extract from web page"
msgstr "Method: Extract from web page"
msgctxt "field:investment.source,name:"
msgid "Name"
msgstr "Name"
@ -466,6 +474,14 @@ msgctxt "help:investment.source,fndident:"
msgid "Identifier found during test query."
msgstr "Identifier found during test query."
msgctxt "field:investment.source,query_method:"
msgid "Method"
msgstr "Method"
msgctxt "help:investment.source,query_method:"
msgid "Select the method to retrieve the data."
msgstr "Select the method to retrieve the data."
msgctxt "model:investment.rate,name:"
msgid "Rate"
msgstr "Rate"

3
message.xml
View file

@ -17,6 +17,9 @@ full copyright notices and license terms. -->
<record model="ir.message" id="msg_err_unknown_content">
<field name="text">failed to identify row content: %(linetxt)s</field>
</record>
<record model="ir.message" id="msg_querytype_web">
<field name="text">Extract from web page</field>
</record>
</data>
</tryton>

107
onlinesource.py
View file

@ -11,6 +11,7 @@ from trytond.model import ModelView, ModelSQL, fields, Unique, Check
from trytond.transaction import Transaction
from trytond.pool import Pool
from trytond.pyson import Eval, Bool
from trytond.i18n import gettext
logger = logging.getLogger(__name__)
@ -39,34 +40,54 @@ fields_check = ['url', 'nsin', 'isin', 'symbol', 'text', 'http_state', \
'fnddate', 'fndrate', 'fndident']
STATES_WEB = {
'invisible': Eval('query_method', '') != 'web',
'required': Eval('query_method', '') == 'web',
}
DEPENDS_WEB = ['query_method']
class OnlineSource(ModelSQL, ModelView):
'Online Source'
__name__ = 'investment.source'
name = fields.Char(string='Name', required=True)
url = fields.Char(string='URL', required=True)
query_method = fields.Selection(string='Method', required=True,
help='Select the method to retrieve the data.',
selection='get_query_methods')
url = fields.Char(string='URL', states=STATES_WEB, depends=DEPENDS_WEB)
nohtml = fields.Boolean(string='Remove HTML',
help='Removes HTML tags before the text is interpreted.')
rgxdate = fields.Char(string='Date', required=True,
help='Regex code to find the date in the downloaded HTML file.')
rgxdatefmt = fields.Selection(string='Date format', required=True,
selection=sel_rgxdatefmt)
rgxrate = fields.Char(string='Rate', required=True,
help='Regex code to find the rate in the downloaded HTML file.')
rgxdecimal = fields.Selection(string='Decimal Separator', required=True,
help='Removes HTML tags before the text is interpreted.',
states={
'invisible': STATES_WEB['invisible'],
}, depends=DEPENDS_WEB)
rgxdate = fields.Char(string='Date',
help='Regex code to find the date in the downloaded HTML file.',
states=STATES_WEB, depends=DEPENDS_WEB)
rgxdatefmt = fields.Selection(string='Date format', selection=sel_rgxdatefmt,
states=STATES_WEB, depends=DEPENDS_WEB)
rgxrate = fields.Char(string='Rate',
help='Regex code to find the rate in the downloaded HTML file.',
states=STATES_WEB, depends=DEPENDS_WEB)
rgxdecimal = fields.Selection(string='Decimal Separator',
help='Decimal separator for converting the market value into a number.',
selection=sel_rgxdecimal)
selection=sel_rgxdecimal, states=STATES_WEB, depends=DEPENDS_WEB)
rgxident = fields.Char(string='Identifier',
help='Regex code to find the identifier in the downloaded HTML file.')
help='Regex code to find the identifier in the downloaded HTML file.',
states={
'invisible': STATES_WEB['invisible'],
}, depends=DEPENDS_WEB)
rgxidtype = fields.Selection(string='ID-Type', selection=sel_rgxidtype,
help='Type of identifier used to validate the result.',
states={
'required': Bool(Eval('rgxident', '')),
}, depends=['rgxident'])
'invisible': STATES_WEB['invisible'],
}, depends=DEPENDS_WEB+['rgxident'])
# field to test requests
used_url = fields.Function(fields.Char(string='Used URL', readonly=True,
help='This URL is used to retrieve the HTML file.'),
help='This URL is used to retrieve the HTML file.',
states={'invisible': STATES_WEB['invisible']}, depends=DEPENDS_WEB),
'on_change_with_used_url')
nsin = fields.Function(fields.Char(string='NSIN'),
'on_change_with_nsin', setter='set_test_value')
@ -93,6 +114,12 @@ class OnlineSource(ModelSQL, ModelView):
super(OnlineSource, cls).__setup__()
cls._order.insert(0, ('name', 'DESC'))
@classmethod
def default_query_method(cls):
""" default: web
"""
return 'web'
@classmethod
def default_url(cls):
""" defaul-url
@ -188,30 +215,56 @@ class OnlineSource(ModelSQL, ModelView):
symbol = self.symbol,
)
@classmethod
def get_query_methods(cls):
""" get list of query-methods
"""
return [
('web', gettext('investment.msg_querytype_web')),
]
@classmethod
def set_test_value(cls, record, name, value):
""" dont store it
"""
pass
@classmethod
def run_query_method(cls, osource, isin, nsin, symbol, debug=False):
""" run selected query to retrive data
result: {
'text': raw-text from query - for debug,
'http_state': state of query,
'date': date() if success,
'rate': Decimal() if success,
'code': identifier - isin/nsin/symbol
}
"""
OSourc = Pool().get('investment.source')
if getattr(osource, 'query_method', None) == 'web':
return OSourc.read_from_website(
osource,
isin = isin,
nsin = nsin,
symbol = symbol,
debug = debug,
)
def call_online_source(self):
""" use updated values to call online-source,
for testing parameters
"""
OSourc = Pool().get('investment.source')
result = OSourc.read_from_website(
self,
isin = self.isin,
nsin = self.nsin,
symbol = self.symbol,
debug = True,
)
self.text = result.get('text', None)
self.http_state = result.get('http_state', None)
self.fnddate = result.get('date', None)
self.fndrate = result.get('rate', None)
self.fndident = result.get('code', None)
result = OSourc.run_query_method(self, self.isin, self.nsin,
self.symbol, debug=True)
if result is not None:
self.text = result.get('text', None)
self.http_state = result.get('http_state', None)
self.fnddate = result.get('date', None)
self.fndrate = result.get('rate', None)
self.fndident = result.get('code', None)
def get_url_with_parameter(self, isin=None, nsin=None, symbol=None):
""" generate url
@ -235,12 +288,12 @@ class OnlineSource(ModelSQL, ModelView):
return
for updtsource in asset.updtsources:
rate_data = cls.read_from_website(
rate_data = cls.run_query_method(
updtsource,
isin = asset.isin,
nsin = asset.wkn,
symbol = asset.secsymb,
)
)
if len(updtsource.rgxident or '') > 0:
# check result - same code?

11
view/source_form.xml
View file

@ -5,16 +5,21 @@ full copyright notices and license terms. -->
<form col="6">
<label name="name"/>
<field name="name"/>
<label name="query_method"/>
<field name="query_method"/>
<newline/>
<separator colspan="6" name="url" string="Method: Extract from web page"/>
<label name="url"/>
<field name="url" colspan="3"/>
<label name="nohtml"/>
<field name="nohtml"/>
<label id="labtp1" string=" "/>
<label xalign="0.0" colspan="3" id="labtempl"
<label xalign="0.0" colspan="3" name="nohtml"
string="URL parameter placeholders: ${isin}, ${nsin}, ${symbol}"/>
<newline/>
<separator colspan="6" id="seprgx" string="Regular expressions to find data"/>
<separator colspan="6" name="url" string="Regular expressions to find data"/>
<label name="rgxdate"/>
<field name="rgxdate"/>
<label name="rgxdatefmt"/>

1
view/source_list.xml
View file

@ -4,5 +4,6 @@ The COPYRIGHT file at the top level of this repository contains the
full copyright notices and license terms. -->
<tree>
<field name="name"/>
<field name="query_method"/>
<field name="url"/>
</tree>