speedup: indexes, caching

model.py

@@ -3,11 +3,18 @@
 # The COPYRIGHT file at the top level of this repository contains the
 # full copyright notices and license terms.
 
-from trytond.model import MultiValueMixin, ValueMixin, fields, Unique
+from trytond.model import MultiValueMixin, ValueMixin, fields, Unique, Model
 from trytond.transaction import Transaction
 from trytond.pool import Pool
+from trytond.cache import MemoryCache
+from datetime import timedelta, datetime
+from decimal import Decimal
 from sql import With, Literal
 from sql.functions import Function
+from sql.conditionals import Coalesce
+
+ENABLE_CACHE = True
+CACHEKEY_CURRENCY = 'currency-%s'
 
 
 class ArrayAgg(Function):
@@ -62,6 +69,149 @@ class Array(Function):
 # end Array
 
 
+class MemCache(Model):
+    """ store values to cache
+    """
+    __name__ = 'cashbook.memcache'
+
+    _cashbook_value_cache = MemoryCache(
+        'cashbook.book.valuecache',
+        context = False,
+        duration = timedelta(seconds=60*60*4))
+
+    @classmethod
+    def read_value(cls, cache_key):
+        """ read values from cache
+        """
+        if ENABLE_CACHE == False:
+            return None
+        return cls._cashbook_value_cache.get(cache_key)
+
+    @classmethod
+    def store_result(cls, records, cache_keys, values):
+        """ store result to cache
+        """
+        if ENABLE_CACHE == False:
+            return
+        for record in records:
+            data = {x:values[x][record.id]
+                        for x in values.keys()
+                            if record.id in values[x].keys()}
+            cls._cashbook_value_cache.set(cache_keys[record.id], data)
+
+    @classmethod
+    def store_value(cls, cache_key, values):
+        """ store values to cache
+        """
+        if ENABLE_CACHE == False:
+            return
+        cls._cashbook_value_cache.set(cache_key, values)
+
+    @classmethod
+    def read_from_cache(cls, records, cache_keys, names, result):
+        """ get stored values from memcache
+        """
+        if ENABLE_CACHE == False:
+            return (records, result)
+
+        todo_records = []
+        for record in records:
+            values = cls.read_value(cache_keys[record.id])
+            if values:
+                for name in names:
+                    if name not in values.keys():
+                        continue
+                    if values[name] is None:
+                        continue
+                    if result[name][record.id] is None:
+                        result[name][record.id] = Decimal('0.0')
+                    result[name][record.id] += values[name]
+            else :
+                todo_records.append(record)
+        return (todo_records, result)
+
+    @classmethod
+    def get_key_by_record(cls, name, record, query, addkeys=[]):
+        """ read records to build a cache-key
+        """
+        pool = Pool()
+        cursor = Transaction().connection.cursor()
+
+        if ENABLE_CACHE == False:
+            return '-'
+
+        dt1 = datetime.now()
+        print('-- get_key_by_record:', name, record.id)
+        fname = [name, str(record.id)]
+        fname.extend(addkeys)
+
+        # query the last edited record for each item in 'query'
+        for line in query:
+            if len(line.keys()) == 0:
+                continue
+
+            if 'cachekey' in line.keys():
+                key = cls.read_value(line['cachekey'])
+                if key:
+                    fname.append(key)
+                    continue
+
+            Model = pool.get(line['model'])
+            tab_model = Model.__table__()
+
+            tab_query = Model.search(line['query'], query=True)
+            qu1 = tab_model.join(tab_query,
+                    condition=tab_query.id==tab_model.id,
+                ).select(
+                    tab_model.id,
+                    tab_model.write_date,
+                    tab_model.create_date,
+                    limit = 1,
+                    order_by = [
+                        Coalesce(tab_model.write_date, tab_model.create_date).desc,
+                        tab_model.id.desc,
+                        ],
+                )
+            cursor.execute(*qu1)
+            records = cursor.fetchall()
+            if len(records) > 0:
+                fname.append(cls.genkey(
+                    records[0][0],
+                    records[0][1],
+                    records[0][2],
+                    ))
+            else :
+                fname.append('0')
+
+            if 'cachekey' in line.keys():
+                key = cls.store_value(line['cachekey'], fname[-1])
+
+        print('-- get_key_by_record-END:', datetime.now() - dt1, '-'.join(fname))
+        return '-'.join(fname)
+
+    @classmethod
+    def genkey(cls, id_record, write_date, create_date):
+        """ get key as text
+        """
+        date_val = write_date if write_date is not None else create_date
+        return '-'.join([
+            str(id_record),
+            '%s%s' % (
+                'w' if write_date is not None else 'c',
+                date_val.timestamp() if date_val is not None else '-'),
+            ])
+
+    @classmethod
+    def record_update(cls, cache_key, record):
+        """ update cache-value
+        """
+        cls.store_value(cache_key,
+            cls.genkey(record.id, record.write_date, record.create_date)
+                if record is not None else None)
+
+# end mem_cache
+
+
 def sub_ids_hierarchical(model_name):
     """ get table with id and sub-ids
     """