From 4a62ab7ce431d1ced8b2a44153ee61afe9a98f25 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Mon, 14 May 2018 17:21:58 +0300 Subject: [PATCH 1/5] Yandex Metrika query runner added --- .../images/db-logos/yandex_appmetrika.png | Bin 0 -> 824 bytes .../assets/images/db-logos/yandex_metrika.png | Bin 0 -> 824 bytes redash/query_runner/yandex_metrika.py | 162 ++++++++++++++++++ redash/settings/__init__.py | 1 + 4 files changed, 163 insertions(+) create mode 100644 client/app/assets/images/db-logos/yandex_appmetrika.png create mode 100644 client/app/assets/images/db-logos/yandex_metrika.png create mode 100644 redash/query_runner/yandex_metrika.py diff --git a/client/app/assets/images/db-logos/yandex_appmetrika.png b/client/app/assets/images/db-logos/yandex_appmetrika.png new file mode 100644 index 0000000000000000000000000000000000000000..736e560517134ff963b1eaf8760a2f47b2fbd8cc GIT binary patch literal 824 zcmeAS@N?(olHy`uVBq!ia0y~yU<5K58911MRQ8&P5Fq6d;1lBddY6h*`KkX54FAu7 zkX782GYkw4g?oYQBdc|4U5dXR76K{&DuYM?#UUa_!@hljQC0!qCAg>jC6&7I;J!Gca%qgD@k*tT_@uLG}_)Usv`=tbBaD#{1iE zSu-#&WqP_ehEy=Vy=l1Wuz>)}g*ghGjZgl$`&zdq%wk>taAnEE6a81;y!wCqu#!Uq z10xd)hkye083`^A?p`y$c0ymT+Vl6n@|yqUH}AP`V_v|yqCr4n<<3<7OjZTvMGP$u zHocO)*{6JgMWzuZzJS|>@deZARplwxeGb9}EKUhao=w}kM6^LqfOP@4&gy#;-&!#F z9uVYcoH#4`+oUjtT@E4*dK=bMxo>;SVZ!LHz~YvbchftH@y>egnpaiNm@cq9V!5-J z_ZriW6h@od->sRn6j-LQ{-^{RFFJuyQ-Ni+gG+;1h2Pv%|7?~S%tlN+PP$A!O-=?Z z(~jP)w07FS#c@zXpvhN3>3|AM6^k)Y1tC>;@`e6f{o2L4fGdUVUXsW<)(_oG6<6Pv zvxGQsZPDBD(C7(A2&1F`6VQefmW*bn4V*eVc&_r8aJaI~V##KcYAtOP$&kztxyI`B zV3B=YW^<^6G!J7gP_=M@_!@hljQC0!qCAg>jC6&7I;J!Gca%qgD@k*tT_@uLG}_)Usv`=tbBaD#{1iE zSu-#&WqP_ehEy=Vy=l1Wuz>)}g*ghGjZgl$`&zdq%wk>taAnEE6a81;y!wCqu#!Uq z10xd)hkye083`^A?p`y$c0ymT+Vl6n@|yqUH}AP`V_v|yqCr4n<<3<7OjZTvMGP$u zHocO)*{6JgMWzuZzJS|>@deZARplwxeGb9}EKUhao=w}kM6^LqfOP@4&gy#;-&!#F z9uVYcoH#4`+oUjtT@E4*dK=bMxo>;SVZ!LHz~YvbchftH@y>egnpaiNm@cq9V!5-J z_ZriW6h@od->sRn6j-LQ{-^{RFFJuyQ-Ni+gG+;1h2Pv%|7?~S%tlN+PP$A!O-=?Z z(~jP)w07FS#c@zXpvhN3>3|AM6^k)Y1tC>;@`e6f{o2L4fGdUVUXsW<)(_oG6<6Pv zvxGQsZPDBD(C7(A2&1F`6VQefmW*bn4V*eVc&_r8aJaI~V##KcYAtOP$&kztxyI`B zV3B=YW^<^6G!J7gP_=M@ Date: Mon, 14 May 2018 17:43:53 +0300 Subject: [PATCH 2/5] metrika: codeclimate fixes --- redash/query_runner/yandex_metrika.py | 40 ++++++++++++++------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/redash/query_runner/yandex_metrika.py b/redash/query_runner/yandex_metrika.py index 98f6556e0..9249adc05 100644 --- a/redash/query_runner/yandex_metrika.py +++ b/redash/query_runner/yandex_metrika.py @@ -6,31 +6,33 @@ import requests from urlparse import parse_qs, urlparse logger = logging.getLogger(__name__) +COLUMN_TYPES = { + 'date': ( + 'firstVisitDate', 'firstVisitStartOfYear', 'firstVisitStartOfQuarter', + 'firstVisitStartOfMonth', 'firstVisitStartOfWeek', + ), + 'datetime': ( + 'firstVisitStartOfHour', 'firstVisitStartOfDekaminute', 'firstVisitStartOfMinute', + 'firstVisitDateTime', 'firstVisitHour', 'firstVisitHourMinute' + + ), + 'int': ( + 'pageViewsInterval', 'pageViews', 'firstVisitYear', 'firstVisitMonth', + 'firstVisitDayOfMonth', 'firstVisitDayOfWeek', 'firstVisitMinute', + 'firstVisitDekaminute', + ) +} + def parse_ym_response(response): columns = [] - COLUMN_TYPES = { - 'date': ( - 'firstVisitDate', 'firstVisitStartOfYear', 'firstVisitStartOfQuarter', - 'firstVisitStartOfMonth', 'firstVisitStartOfWeek', - ), - 'datetime': ( - 'firstVisitStartOfHour', 'firstVisitStartOfDekaminute', 'firstVisitStartOfMinute', - 'firstVisitDateTime', 'firstVisitHour', 'firstVisitHourMinute' - ), - 'int': ( - 'pageViewsInterval', 'pageViews', 'firstVisitYear', 'firstVisitMonth', - 'firstVisitDayOfMonth', 'firstVisitDayOfWeek', 'firstVisitMinute', - 'firstVisitDekaminute', - ) - } for type_, elements in COLUMN_TYPES.items(): for el in elements: if 'first' in el: el = el.replace('first', 'last') COLUMN_TYPES[type_] += (el, ) - + dimensions_len = len(response['query']['dimensions']) for h in response['query']['dimensions'] + response['query']['metrics']: @@ -46,7 +48,7 @@ def parse_ym_response(response): 'friendly_name': friendly_name, 'type': data_type, }) - + rows = [] columns_fixed = False for row in response['data']: @@ -58,10 +60,10 @@ def parse_ym_response(response): if not columns_fixed: if isinstance(d, float): columns[dimensions_len + i]['type'] = TYPE_FLOAT - + columns_fixed = True rows.append(res) - + return {'columns': columns, 'rows': rows} From fcafc40be3373c39054ee9f1461729acb48f4b25 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Thu, 17 May 2018 10:58:32 +0300 Subject: [PATCH 3/5] yandex_metrika: codeclimate fixes #2 --- redash/query_runner/yandex_metrika.py | 37 ++++++++++----------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/redash/query_runner/yandex_metrika.py b/redash/query_runner/yandex_metrika.py index 9249adc05..5ae1be64f 100644 --- a/redash/query_runner/yandex_metrika.py +++ b/redash/query_runner/yandex_metrika.py @@ -23,18 +23,17 @@ COLUMN_TYPES = { ) } +for type_, elements in COLUMN_TYPES.items(): + for el in elements: + if 'first' in el: + el = el.replace('first', 'last') + COLUMN_TYPES[type_] += (el, ) + def parse_ym_response(response): columns = [] - - for type_, elements in COLUMN_TYPES.items(): - for el in elements: - if 'first' in el: - el = el.replace('first', 'last') - COLUMN_TYPES[type_] += (el, ) - dimensions_len = len(response['query']['dimensions']) - + for h in response['query']['dimensions'] + response['query']['metrics']: friendly_name = h.split(':')[-1] if friendly_name in COLUMN_TYPES['date']: @@ -42,26 +41,18 @@ def parse_ym_response(response): elif friendly_name in COLUMN_TYPES['datetime']: data_type = TYPE_DATETIME else: - data_type = 'STRING' - columns.append({ - 'name': h, - 'friendly_name': friendly_name, - 'type': data_type, - }) + data_type = TYPE_STRING + columns.append({'name': h, 'friendly_name': friendly_name, 'type': data_type}) rows = [] - columns_fixed = False - for row in response['data']: + for num, row in enumerate(response['data']): res = {} for i, d in enumerate(row['dimensions']): res[columns[i]['name']] = d['name'] for i, d in enumerate(row['metrics']): res[columns[dimensions_len + i]['name']] = d - if not columns_fixed: - if isinstance(d, float): - columns[dimensions_len + i]['type'] = TYPE_FLOAT - - columns_fixed = True + if num == 0 and isinstance(d, float): + columns[dimensions_len + i]['type'] = TYPE_FLOAT rows.append(res) return {'columns': columns, 'rows': rows} @@ -100,7 +91,7 @@ class YandexMetrika(BaseSQLQueryRunner): self.list_path = 'counters' def _get_tables(self, schema): - + counters = self._send_query('management/v1/{0}'.format(self.list_path)) for row in counters[self.list_path]: @@ -114,7 +105,7 @@ class YandexMetrika(BaseSQLQueryRunner): schema[owner]['columns'].append(counter) return schema.values() - + def test_connection(self): self._send_query('management/v1/{0}'.format(self.list_path)) From 8422a3657ab7310dcfd76b461e048a819843c313 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Fri, 25 May 2018 16:56:27 +0300 Subject: [PATCH 4/5] metrika: added yaml support --- redash/query_runner/yandex_metrika.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/yandex_metrika.py b/redash/query_runner/yandex_metrika.py index 5ae1be64f..a93161b41 100644 --- a/redash/query_runner/yandex_metrika.py +++ b/redash/query_runner/yandex_metrika.py @@ -1,4 +1,5 @@ import json +import yaml import logging from redash.query_runner import * from redash.utils import JSONEncoder @@ -86,7 +87,7 @@ class YandexMetrika(BaseSQLQueryRunner): def __init__(self, configuration): super(YandexMetrika, self).__init__(configuration) - self.syntax = 'json' + self.syntax = 'yaml' self.host = 'https://api-metrika.yandex.ru' self.list_path = 'counters' @@ -119,13 +120,23 @@ class YandexMetrika(BaseSQLQueryRunner): def run_query(self, query, user): logger.debug("Metrika is about to execute query: %s", query) data = None + query = query.strip() if query == "": error = "Query is empty" return data, error try: - params = json.loads(query) - except ValueError: - params = parse_qs(urlparse(query).query, keep_blank_values=True) + params = yaml.load(query) + except ValueError as e: + logging.exception(e) + error = unicode(e) + return data, error + + if isinstance(params, dict): + if 'url' in params: + params = parse_qs(urlparse(params['url']).query, keep_blank_values=True) + else: + error = 'The query format must be JSON or YAML' + return data, error try: data = json.dumps(parse_ym_response(self._send_query(**params)), cls=JSONEncoder) From 810107737139cb362ad2fe230f2eb3c30bd4e9e5 Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Fri, 25 May 2018 17:29:53 +0300 Subject: [PATCH 5/5] yandex_metrika: new appmetrika logo --- .../images/db-logos/yandex_appmetrika.png | Bin 824 -> 1459 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/client/app/assets/images/db-logos/yandex_appmetrika.png b/client/app/assets/images/db-logos/yandex_appmetrika.png index 736e560517134ff963b1eaf8760a2f47b2fbd8cc..764e7cff2601d7b7bad966f0ae601b2bb9b96c83 100644 GIT binary patch delta 1329 zcmaKp`#+Nl9LJy8vYgz8c$P*HN{bDh7VSt#PJ4CZAe6_Zi>@S5Q_pO(PNo%(%%ODB zq?4C%bl8dVpt+4xy0{#2nOrK@l#R0Itbd^QkMGy(^Ll;0-;YeUeA)Fi2EKb3fxc)f zuq~kl1=8KF#Dfz(JsG(aV}7J`=x2EtdsGE? z;#gC4Mq_;N>JB*D()E21UwPLtTlnjD8UK77vA)$`KDXl1G(Y70mEC!IRi~7^dq@T$ zsXAdin`7(VaO^%zyBAQL#Vfz?bXN|oWYc&6$Ch2^ltQbpB=1Y(eL>8tViNHokq#o2 zXkIn+!xQSo<>61M+ogPkOg`{BX7xR=%EoEf#5-6nU&(yiBFpp84<}ii6~r1Lb_XK6 zWV z&~q0er$Y_086ap*p%p(`HN<3o6*iC7k;X zGlnjugmd?!eF>a)liP4Jy~)gg8ru7H2{>)42Nn=@cOna&86-$_7$i>$C#3Wm9GsTL z8O~S#L9>tMmnlOozM3FFe-;fJ(3GP}P@*SDWzC%a3YnF=)$ZAQeAB>M)D9Axi`S+8 z?VxU7D~RT#-wCTN9 zv|lz&mr=I=C3o{8KeC&$idqxKFwHolO+^{ob~ArB?$947Se+InjMjOfP5n!6+EF34 z=h*SoF@S9jQw#*@D`fbbX0!SeJ8Lk0NtY~Lwe}<#lw)#X@i)zUEPKnop2MKF-srV} z5Mr3P`BkTF+hgF!iAehZ_@@_NooZKGikkzKx}~8uqWz7G!iN|}d8#}k_POI>m#l}F z;w({S5H)i{;r{wiBJG&Lw_eN1St<2Flnkw8O19)xaMMpv+f3FUQp)VU9kY!5n{XDs z0XCbzB$w{_(eQ3XM2fS~2&Q8ll8*Q^*jOSib~ID5_FJuG41(cJ)~e&JuJw*7v~z;w zV(K&fTr*|Y{e}ccZ6Yi4UuSp=vSKqn@iw-^e#8xRvF=zGmtp--Xr)$OwL3RBT{#kr z9Ae|UA}tJImUhE;lK-?T!)gmbbaKvia;e4WwJp^cOAkbc{YHkE7CfvC#EwWgJChPO z0}uLB#7}gbW(J3WViHCmK}pL=G~WP%-?sEAX!9nt$ZAaZlgFL( zCgk<`3DjLDEv!F|2sn6GjyzoG$uj!!Yk%pqT#%nt-|Z zF4gNw`wyU-#&gimzn7O;VNJKmjq!;8QrXBF{!lxii8z}-Kid#tO!zL%A1%66F9_{i R@$v#di@)zKpOS45_kRUSfnWdt literal 824 zcmeAS@N?(olHy`uVBq!ia0y~yU<5K58911MRQ8&P5Fq6d;1lBddY6h*`KkX54FAu7 zkX782GYkw4g?oYQBdc|4U5dXR76K{&DuYM?#UUa_!@hljQC0!qCAg>jC6&7I;J!Gca%qgD@k*tT_@uLG}_)Usv`=tbBaD#{1iE zSu-#&WqP_ehEy=Vy=l1Wuz>)}g*ghGjZgl$`&zdq%wk>taAnEE6a81;y!wCqu#!Uq z10xd)hkye083`^A?p`y$c0ymT+Vl6n@|yqUH}AP`V_v|yqCr4n<<3<7OjZTvMGP$u zHocO)*{6JgMWzuZzJS|>@deZARplwxeGb9}EKUhao=w}kM6^LqfOP@4&gy#;-&!#F z9uVYcoH#4`+oUjtT@E4*dK=bMxo>;SVZ!LHz~YvbchftH@y>egnpaiNm@cq9V!5-J z_ZriW6h@od->sRn6j-LQ{-^{RFFJuyQ-Ni+gG+;1h2Pv%|7?~S%tlN+PP$A!O-=?Z z(~jP)w07FS#c@zXpvhN3>3|AM6^k)Y1tC>;@`e6f{o2L4fGdUVUXsW<)(_oG6<6Pv zvxGQsZPDBD(C7(A2&1F`6VQefmW*bn4V*eVc&_r8aJaI~V##KcYAtOP$&kztxyI`B zV3B=YW^<^6G!J7gP_=M@