GtDataStore/app/cleansing/services/jobs/source_data_set.py

from models.source.opc import Opc
from models.source.mgj import MGJ

class SourceDataSet():
    _opc_model = None
    _mgj_model = None

    _last_id_dict = {}

    def __init__(self, config):
        self.config = config
        self._opc_model = Opc(self.config['scada_db'])

    def fetch_increment(self,limit):
        result = []
        for i in range(0, 40):
            table_name = 'scada_data_{}'.format(i)

            last_id = 0
            if table_name in self._last_id_dict:
                last_id = self._last_id_dict[table_name]

            data = self._opc_model.find_increment(table_name, last_id=last_id,limit=limit)
            if len(data):
                self._last_id_dict[table_name] = data[len(data) - 1][0]
                result.extend(data)

        return result

    def group_by_items(self, data, filter_project=[]):
        result = {}
        for one in data:
            if len(filter_project) > 0 and one[1] not in filter_project:
                continue
            key = '{}.{}'.format(one[1], one[2])
            if key not in result:
                result[key] = []

            result[key].append((one[3], one[4]))

        return result