From 81e492297d22698c55eda6c64c116eb92a338f1a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 15:21:03 +0200 Subject: [PATCH 001/540] Link to the license file in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0831cf684..133358552 100644 --- a/README.md +++ b/README.md @@ -45,4 +45,4 @@ TBD. ## License -See LICENSE file. +See [LICENSE](https://github.com/EverythingMe/redash/blob/master/LICENSE) file. From d1d4510768d42c816055a0b37c251c404a0bc3e3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 15:23:10 +0200 Subject: [PATCH 002/540] README: fix link to new issue. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 133358552..dff162cb9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ TBD. ## Reporting Bugs and Contributing Code -* Want to report a bug or request a feature? Please open [an issue](https://github.com/everythingme/re:dash/issues/new). +* Want to report a bug or request a feature? Please open [an issue](https://github.com/everythingme/redash/issues/new). * Want to help us build **_re:dash_**? Fork the project and make a pull request. We need all the help we can get! ## License From 306a1646cfb1c304710c182cb743bbbd12f47dc1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 16:51:49 +0200 Subject: [PATCH 003/540] updated roadmap --- README.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dff162cb9..e7a9f4608 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,28 @@ TBD. ## Roadmap -TBD. +We plan to release new minor version every 2-3 weeks. Of course, if we get additional help from contributors it will help speed things up. + +Below you can see the "big" features of the next 3 releases (for full list, click on the link): + +### [v0.2](https://github.com/EverythingMe/redash/issues?milestone=1&state=open) + +- Ability to generate multiple visualizations for a single query (dataset) in a more flexible way than today. Also easier extensbility points to add additional visualizations. +- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors). +- UI Improvements (better notifications & flows, improved queries page) +- Comments on queries. + +### [v0.3](https://github.com/EverythingMe/redash/issues?milestone=2&state=open) + +- Support for API access using API keys, instead of Google Login. +- Multiple databases support (including other database type than PostgreSQL). +- Scheduled reports by email. + +### [v0.4](https://github.com/EverythingMe/redash/issues?milestone=3&state=open) + +- Query versioning. +- More "realtime" UI (using websockets). +- More visualizations. ## Reporting Bugs and Contributing Code From 6cf580e0ff1037527fce945df9362f84d76f12cf Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 18:24:41 +0200 Subject: [PATCH 004/540] Add resolutions to bower.json. --- rd_ui/bower.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 1af7dafa9..b7ff4ed63 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -21,5 +21,9 @@ "devDependencies": { "angular-mocks": "~1.0.7", "angular-scenario": "~1.0.7" + }, + "resolutions": { + "angular": "1.0.8", + "jquery": "~1.9.1" } } From 604a2946620f9d8a56676eb9b4b647a45ef7278f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 19:34:40 +0200 Subject: [PATCH 005/540] Fix use min configuration not to depend on the bower_components directory in dist. --- rd_ui/Gruntfile.js | 2 +- .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 14079 bytes .../fonts/glyphicons-halflings-regular.svg | 228 ++++++++++++++++++ .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 29512 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 16448 bytes rd_ui/app/index.html | 5 +- rd_ui/bower.json | 3 +- 7 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 rd_ui/app/fonts/glyphicons-halflings-regular.eot create mode 100644 rd_ui/app/fonts/glyphicons-halflings-regular.svg create mode 100644 rd_ui/app/fonts/glyphicons-halflings-regular.ttf create mode 100644 rd_ui/app/fonts/glyphicons-halflings-regular.woff diff --git a/rd_ui/Gruntfile.js b/rd_ui/Gruntfile.js index ab51fd0fd..374e0cd6e 100644 --- a/rd_ui/Gruntfile.js +++ b/rd_ui/Gruntfile.js @@ -249,7 +249,7 @@ module.exports = function (grunt) { '.htaccess', 'bower_components/**/*', 'images/{,*/}*.{gif,webp}', - 'styles/fonts/*' + 'fonts/*' ] }, { expand: true, diff --git a/rd_ui/app/fonts/glyphicons-halflings-regular.eot b/rd_ui/app/fonts/glyphicons-halflings-regular.eot new file mode 100644 index 0000000000000000000000000000000000000000..87eaa434234e2a984c261e0450a2f4ad837aa7b4 GIT binary patch literal 14079 zcma)jRa_K6^zJUrQcHI&-Agwt-Q6i&BGL^KOLw;{-AD_FG)Q-gGzdrvN-EcX-iP~g z&*b^eH{Y4xyv%PN=0ykqC=mnzkp2}Ez<(I(fA#{~JL1@9|&czbr17 z?0>QUi2(qt040DrzyzQTPzI;~05<^oukZrI|7re*(tmmX7j^o_^aj}eC*Svf zS8xM_|1re@Z~iI2{-^mL9EX2e|B>GY!1r$^_@7M#!2iz^{g+$h|9j_j|IfYw09iey z|2e7uJq%=kUm`%z3m_N(;2I^EK8c@Rz+WzA_5K>K_A~&N-y3An#=6kB0L1`ghg@hn zZl7)JRrzdfN4}^l((rOb8!6cPsFL3<+h>Ko$*N(B`~JnKcb$DjB~XQQFl-maOT7?| z=??-O{TBG@KcAzmSNxsJz-Lt-`@AJr0kN!Di;SF6C_P<|x%6Q{;498Vwc}wHl?UCr z{Q~3fpz|ayjwAvkULRl`8oaqCD1Wz4@8$~fj$UC?mYD}9H~K)mrxoe9!WwG7+6D1~ zu)}%fLgSy{-z-;>e_xUdTzZz=OI{SZWnRf9!Z!c1f25WUO+5X9vri&A$czeCIfk$M z9$(eLNbUdRcqZ=w)1@@tN<^z0pQP-fOfjvjK3hvorqiV%Rl2xSOKU%hzr6ahgV9*$ zJlgSvPU509MBT=C+`yifpkEyy8#9c4UL5|r5gWS_tr}Av>(G)ZhAtjcTRS3?SSA9N z_Kegnh`V2N6RU=69p<{&He6g~O%EZ5+2OH{@ca1ru$Z)c3E&|1G!5~|4CfxK{)bF7rn^i` zwcKpWlzAHWR{;3USb36)e|%;$T55rp9tZ<6==s|-B*BebGk#$IYB|(ZrzrewrIl2Q zcVZsN=FLe{6k5m7YDaR%(#gdFf#BlrKVjI$R-nNKpd*2(T6`_?7Tr%rq~E9(yIypk z15x#%OfK;;uk|PQR~)DEppbSH6DmW;v@k*#ZhaG5{w7e$S`ot*K<^C*oB^co5cNr- z84k3(uHIXMy>++r-IRV%?Vpo$*r`8)jmh{vx(My9BI&4V4t z@q&H_L`zH3p725(a{oTG;rYk3%_{r*|8>5_6G?cTr)|U^XlDg8z zm^W6r3{qR3liJadUw%-DfiMsiV2YTxYOPA_X1lBkNTo&NjbQ(_zP!Rimikpp%G~h_ ztU^LLtxb8e!>D>CG^8eZ_@-EFi+JA&%Ym}4^tY?&sz92_hbFAune34RX{tbjogYXK zb;~ja9%4IE{_iiY6WdJ>_PH&3&@yDo2T(p1E`%?ub^PQ3)diW6ii}#+*!=`BpbGP_1R+t&;29S$UAcpH3h}2^>rGvH){c0jJtjcaSiIpFl?|Ykw|FXrNy% zn~l3m7e4&RgrOCH+jCRW=Ls5PATEyA`J8Ad?TVOG`l@pE({KV)pF3Z7;oa4-Hx3nk z^j1RZ{N?bQZy$cYv6=A&0^)qVweZ{+Bno|~E=9j=k-GDXeQ3qsW?N%I&@}1?wxuHf zA|Ro-_+d*C6M-#@VpM30RTEPdo!APpRrFObUDP^Ic|AJ;)&LVdnWX#RxiFb+zGKCQ zI_Kger%ADWvepR*8TGZ{JN(1K9%&P;^!XU4tSvkgGe_{JR~^f9$<0Tklc96r9x1B=VltaV_PCB77l_0tL3{`BdedCe5j3CF zO*e3HwE9GE<^LnU6k=*E%b)otxd+9+t<9)#+ze$kGPmX41&oF?8tHV!$ntX{*8aX^eeP@F2xMvpFGcra42@FI zDr{tW)yt3)P*7pvoD&$N2UDat?KH#6Zr3Wj1ocGNeW7Gj^2e)tH;o4O)FyAx_b=b8 zd=9(x+S@-Ai=UJC?i@DuZ0CtTtAU!S<4~e$K4CsxC85Tve7fHoj%T!vPv{JHch5_Y zM%K`rC>1Uk_m|u`%z4L~W*R<1JgN zI(cyXr))hytWI9~bat*Gf;?_avFr#*aq=$;3DEl;rBBbSfL&s-CmEN9Z=FWBPq|*w zV=1XfmME`nZtgN@DBWrbTSnz2oWcA9yL*=L#%fP3TXt!c0F%_>FvWM9H}5Urg0WkI zNt&dRN)2J@03gGYXLU}Ws1SoLa(2xNG04O@u`3C?42=UF%K^ZmD2OcrLpkyPD{zkZ zqZSrZ%U#vZMaTD{N9>OdGG?lPL;z?aQq&oxZHacwkYDWEjRc9X)Mg4w1*sqqdytQc z;>DOou1OedrNNb->@o%dNQsBess9-iEOg6MCTz%8RuuTHw%yfj66ap};<tL)BjF!!xYDU^iC@^Rt2BMhA>^Oluv#5vBd^doV(|U*_eW!Fpo^kadb~1qfM1 z-4xV$$`eWJMc%3OjU5A{fCA-11x&T35;A``cBD@_K+AfYp`ItY-nO9GFXyk(6H&gC zgVP-%-^o=btFjCC^slGFm}WC)1Fkw6WT{3uKjkNm`0Q%U67%Y#OLYbxB}u8qEXyBf z+jt?k7GWf9V1;7X7NJF^$kk!j@XFwhY;np}TTfKNM)sdEtVZLgSNz~z0}w_y_MM$P z{7ZPot7f{~deqdkb!?PO@3M6uVpZ)~0PM!uFW*8tGxGouYU+idM&+mch>1YWrfYbw zNHh7S!OA3^0A)hxl7xkSusWMIn}pAG7sVY<1G(8sqQS{%57LmXJp-HiSyD=l$*Riw zY+20T)}-|#pikZ7^U!gc1p%vkX1Q*!C%Ns1AbUha>5MtQHVJ(Q7;^mZrN_`4&gR#d z*GMiPozmbFnk7GQMUfb1z-LiF4xQ67RJ<1As!AEvs7ht4PG7P&xpL)JUK!S%jeUiX ziGEQ1j5YCz%;X#HVS2_}6~%)EQ*SZCzV-TqZo{O6%{r8|Py{vm3>zZHrnDT-D+S?Jo!n<`QZ%7N z6#HY((OAs1v%<)LZ%T1o@hclr9U{s$FY2`$#A222+iwA0^_ZWa}Sp$~Z`tSRz?fYd)Prtgp>DC@x&win* zYx)}AGLxzuz+^6ox_-KQe7OJaF4>UhEn2<^kp=1~zSKf2O8lsvgwt(+%dH&YE^$~{ zmIZuN4KWfnT+eLo`$Ntu+@_4dx-xCn%;H+*qI*rz{Pj+IMWV4q&4&v_vDJ?KnuhT? zp`HFH-{i7G z&cb3tRVzJC2)Aj&v-_2I=-cTnDad;U%gi?|r{%q8M3=JWIA4A_$1xksNX8fGQ0MXv z7jsG@yqP^YVXh~FGG7ztRofbb%v-Y2Oa0c4{DoEW2+ghB#=X?sC)zOnd<$FcA;P}k z!&0wB1tjlcu)sC=F=AuzvQsD3oXvch4Ur;5+K@a2;bjf`X@%InJU~*7p!QXL|3UP=)q(sV!;RVRF4eC( z5w2y7m}t3+flB}{o?fK>I$D|ykMw@kZumiw3J18$_+UA|-{#xqT-R~i?db}=&OhR9(;d>s&5GJ-M zuHl@XB;EHQ^c`j#mM47s|SScy-SD&Q0s(780*ui5*B(NU{ z1JAM6oymA%{(T`Qwoer|4`e4fbXpw=Ujf|X8hmq7E&vxv*}=+Rye%5X2xD0*^}YEf zEGd7~le2mpyS%mw8xl44hIvof|Pxp1T*z47AL}K^XlL>J6(gyYOmc|;VYs(tHAWpG7 znr9Tel(H$KV%()2(VBNVoP!o~|Gd)(^S&Q{PCqTk&dV;xZm_-lB_hr!QE$$#GqKT6 zV~RS4<7x-=tx0m&jE1BDqd(cc2iA@B7Ib0!{b&v`-5`t7XEV6UG7WdVy)z(@VR3p< zDC1lTpXHX3oE}5E3V7yx^8>jVnwr!w1_he&_17RJW+}R?{niZFG|4RyT7ZmC!Y^% zbR{57inS^QNGx!}+P3f7%?Sionp@*#h+8;FTaj1>q z1~X!#NO{YL-6+QR)z_o*SW%A+v-XebXs8&@TRzyDRieHy_t(B}bl)uwdFg%YXZ-^# zMWTYOwIkzv%>xr%$CBM=*m$T9k}!UxqnsS6rl-gw-*rU&V2or^ZkP6vPI|0njAB4O zn5CyBPHvXL)29>zpPkhW{`Qw3B?(G-TWfAV0^+}Ji$*Wob6n`WzRTBhd{);=mfm^% z{;`v`S>9Z(j2Nv-VLKD3~iA$Oj{Dq0(I z8U*-!Po9%GdOD|LVS~3(q-_)biNZxTiT)GN)YVr!4f4IRLNhAD48qw@0S#E{-e>UP z!dWH9**gQ$DqT?TkKNJl#J(f~7r6JAfSveml{UZ6jueeC&zR#Vi@e*Z==rWJgp@xj zDdR~Hd=3W?q0l(VMfRu(XreTXK*$pogtsuagZUmp^U^=wp0PM}Wf8W^Fm9n^8S4AS z7GJfQqzDgu-5C9o_f0zKKx$9L$|nGrE2rf%PLxV|c5LZ}PzELiSVok_zxZdiw78@4 zczsV08yXH>t5P&u(+XYPsiu48SXe7a3yEBGFiS7KFN#T`R)LMID_lZrUwvIx-Jfbw zW&lwFFkZK~+S9BQcb`8iqN%$0O{ zd_R#~i~MUF@fY!H4LxF+H=SJ{%h^?na-7Yogv2T6317oP^NJ}Jbg&)D&P;P^w8oe# zDNHRAqcPe>x zP|B*V4YPfm)deuX7-N@-7Mz4N1KmAfyYI78#jS0>Bkd}i9TWLsIZgXQY}1jqm+pG` zy{JiBImlPiF($3(sE&p7ntgNWLh&&5y{|mea7L8%c);7R2$T z_HrZz(`Nx;xE)NtPgF(IH0m#(y)Npg}NBkIWpJb(OJq&ymq^iBIHfZB+V!qd}3EnxDKf_XvD zT3tuka_2>|KJ_Qr(qpGJAf}w3%5Qo=u)K?~`O2CzZnMD_J96QGYE`74E@)I~ODsKK zH%}vL(dJC~ZUF3t99-z<+)r4yfgnU{Y-RryR^-SYY95;xsg#!aUC-Afy-0t%`Ccv_)YQ)A}F@oIMmu2ZX7PQ72ukwf(Cvsr!%uk z?~fxQtYEo0ehCIE`*_+|rxqV~hPV#FQyC(#HP&p@G#fKOUMp?w>)uN0&^pgnu4xwA z{+=Wo;`6mUi`y&O^6j1|StaDJHzuv-uBNf~cik{Jl#-tM_hJ^k+>c0kMduSMRtVAB zXTfh&yMOb>MNO5I1PZ0o!i;G4!y_^YHKHq6oX4a^KR@ocvM24QDH>)gQ-zdAXg{pR zt7?3h$uSFFv$4~lRcBSlUCKIO9p9VFeN}^EPQrbB!iSk~Ba2aSpMlf7sUnT!2PnKp z*Z0Gpr%sIM*x*BP?6E2Zk^y$a@Bl!Rt4YArYn_Po5M;&@gJz097wEglfz`ESLsIET zBs|I>ZJ0yIG}&DmAFB*@>{;;yJ_vO?f1N3M;xsLT(}SOFekLA$9KWf&-oNL?8X4J4oyU8tKa|1>*wEyh6Ebf)U!Z zYdS#`zoaL-RrPmx!}8501YZ{qj!4m&Y7SrdF&73udbUZylkG?gV+qAaszsvHEe+{D z<45m&hYodO2}g4E7>W2VeQ&n7!#30RJ8KbdK;T;5$lg`8J^y4jw3DP%j^Drg_woO{_t+eT$A)(~X?aCV(oI(=tpI1st*S@&~g6?&k z>s|?NRJcDff1`1?-Jc?K@U3-!Ys+&;g!A9IYGA|)zLH&vmifA**}mdVQFo{e8U~b2 zO2E010oyxaVfzV>!DiaH1em79k8chs%8c=txP&UaPiGwS0WcWl(|%w+^T*t*H|mk8 zz)Ak3o-PR;*!0I#w>D*9!+3J9$A|8=Ap!W>(U}g$h&Z!YOggAp^3=wF!Yaz_P($@? z(n!BM5i+f_^FX8~nrY$)=ZBTKHqm zVdAIS4fs!QL{-!F1~xy(})Hxa6p?Rjwv#-#Pvf zm8TQQeBr%Pn(2S+vFpu&c%{Rrk4#{RycSckZsn7q)i-C?s^e~PurOnw~O zv`sbAk*TMuA3Lo&9S}C+NVe+lL`zRzEuw^L!#*K_R{1j-SsyFUDFnW}3R%$ zis0vASSvzW7Jd2#61)h4#M6URkA_A3SsK4n#`cE2$ zLWp@8V}aGF=zO!}e(^Si*LlMGu3Si8)@_u+nrICpR-ng^i~GNd$UP_6*gd;57I81d zqLuuFat(5+->FEsY>{47M=^M$XX_r^DhHhyoVF&%)642YK9oHn`28XL@oD6zTRCr_ zQj#&uvxDDr@MK}Rs%^cX(zMsDRa3RzUQqW?O#N@x@1442leTwu=(D`c&~bPJX1eJx zR}5A8N$9Bq;W2HP`r4=%i4+)}>MCN-g9+FaIfz4#pX3o%gk8jR#?u%4F3+u2WCA{+7b24rYuJ1 zwW3Y9w-Bt2a(91Hcuj#xdB*q8Hy&$|)<1KPvN*|iiK~tq?ka$u;jeH>1QR}^dUxIFtyRN6z{I4L_o?enJ zFR95EMp$tQTUr!1vOm|XcjELh%@1qHj^++_t7XehC^Kxgs_HUQqFOBndGbf*;KnrP z>1BrQ)f5<&={TbN%QdERb6ljEbbCGjdd@5M#n06;VPP)$ z>chCAA@WK55n7o^L|)RL4<9m6lWth#q>&#GG5)ftZ#UzvbU+$2(jP)!o(zaw#;sdv z^%g(${-K@o670tu4>IZELt3#`+>9j?qf(`5Ch+>S&;~QQKzkSNY)16RqV;^f>T9$m zdqgaB84{#YEI4zWG)0m2{JP4snKf5{q~3>X2#QxOjG=sO9EHimSic@4V^<|@R-5Hy zEp^BF6R52jd09ovYpsaxywq*xnqd^%9fxrz=LFuUgxW6tSBC@dGWefD{H&>5oMjlj z6Ud@Q2;X<$!M}!W1R~uQvtTfS6QH%6nlH&~+q&RAWmVP$rbyZI&7MJD!MWh1sb*t; z&V+sSq(hi;g5~PTh!VqP_4Zlgx`%k?t19FqAJy6{$9?t}qv_oZP(+mjL!&s9hsSi0 z`1hZBgO1QyH=#|A^)bdk-w<5x6J#hivLy8_sDXLZ9cyp#>1cVkuO~R8$$=T!YcnR* z2IK3z=tD9$YM0E;xMYvjGX;DYEKeMPAY0k(Lwzo{Vh7}c15$J|s~_D_e%+RH^Zh!m zk4lp6r#OascmM8jGUcEAXfHU(neLo*wABl3)3I;N>=s`|zJAWwZHZtQNH-HR7WUvwmZrG!N z6@C{M0eWXL%2LZxW5tb=HS-8XP81s4JBB@;v&wkf0l#Qa_S5T7lahYrpP#_4z4ku! z%79{Wf8-DjEOK`d7PC)LJqBs(n-#-j1cvFr54a3Sabtu+VZ|9mz#=H?Or~eqxl$PQ@(j-#K-^vA1?!cVSYHiqjG%wgoo{ z;V>B_%aMBK*fx*zO(E~G2V^Rge0k6DE6)El91p>sh#YPjHEIdf%#qo8d;2q;-PEL# zM$qSYuUAeQ2&IGK;PK6zotMsO$LC!pl>@QKlp--=jQIkEwD||8ke1rQc)#gAZCdSP zbp|sBqb`OyD=c13US7+@&9PO~KE57bfoh^{0jOecez`2lpKQh@(KW*IF9t5p(vD6; zqC<&N{Yb0E4bC_{JpkUsO@rlnQkGCgPZc&=!#+=sq3)AE1cd=a-Lo&kH67=u3f~^x z$gvF;{hY5N=zW-MGNTT=kuvj=Eeje|_OvDefcre>sl=DrFKM*}wkk;l`}4haQL%D& zozLBx7UB^7A2;9x3fXkFDG|nU!vVTV#n;l`sA<8?C44E$S_CvCJyIKcbBTSJm2-dp z+A@d77melYFx?WF=8D}pZGaBq7o{5e+?i$`$d&UL1MLb{9o$$YA(U~As5FJ(o8zOW zjycOOtBY}?CJP+$sVEXp?BZ2aL1i4K0obmwIcc&4(62jbW8swa9f?DjTSetJS_F2B z5Z$cKkvqo(>(e|^<$|2NpV%tz7CM|Ai^m?Kd>Yu-{R!v%f8RBr7rWNtfZ^9vKm!u^dP~TR}A-E{C@XK9TX7!)BcW+IpovW>PA7tEh)jxk?zJUM*2{Y zN?T}i@F{LR5-+vp%IKQlcB3Ym)7}cJ12(U+D}MPeLlGDyvcfbe8%LPEy)G!?=e1L= zDJJoWSy{8;p|+#$)~16&EB2)`e$!tX1y-N{WXm?gwG*OnD!ci3u-9+(iLd7=7;7jR zmcY=*?xB}|#asYF%EX6t2{+RK&4M4{66KihGOAs;ij@mK&3Uu)3^b|?B;3B+z!38I z93x_C6}@3&mJvH)!lIq0oQQL86oWy_A|U@GvyD(NwO$c!`%U{`)TMN_Jau#t*Y0lu z0c4~`*Vxk$tP&+W8%8kVnREOkJevuHD;AI8ltWOEzPR%_#f5(Y$jArOxfd2TY42x( zvdviv@hBSfQLqM3;mpaTz|811VlQ7jQEm?Is1NzX>fhX*)3?iglf#v5#%li7DBSDs z9yr*Son&|AfaSp^FHcK!iyS|rW|~Ho3BGnwfGSacSD-Pd3HZx4^Tn{rw@X)t0G#!L z)6pFajr<=k25R8M>3^D^?Vl5V6+B+5p3Y=}-8meaQr23s5Ci^QiE_I#JND7F{`x)Z z${rPtj&q-)Eg1mQ&R^d8PLmmpTs0_NfM;Ld9p`~M`3B|`d)KSkHhIgWGh4h9V(M!E zprOL?IrlHS-Zj#5YaezY^EfJop++5!6~dG@VczVZsShn@a!H)^)mLap zN-5d|ZA^-9-}C0NQY-(>WWq2>z$nZ#9f)04o}#fdrZX(@%ws*mvWvY{x|!V;M+h(u zc(X?j+n3l}NT?SeX>yk#wP026HlrMO$^jJSY9}JbsQW`La`|uCRVgB?-NUkr!Q62rlZJ0 z4(P@;r`r%R2v%XcY4gwA4RY5cS9^>;1!-;WRHH6?A9H4nS~L6+Erf{kNRARp0%v#mG!BN`{Z0DT(;hL>q2tUur3n4FyKJATTZeC)I7~MlF{vYq zP#u$a?65CY1gX<_^dpm$T93g7cEiaEzJi=f(PP7*$Cf< z3e!q;mMXoy);Hc=X!%VmT-e!^igX6GoDK`Lrz#=>sc zkvcN?I-(oNR%$y<5v;+H$CX{e0F$s;-Dc+ckzFlEF7xK<7+Ij5F~FWrmDWsXraDch zDC0G}@xv|q?bH-m|Mjy0Ms)dZNpHw-DvLp2+c4S+O0)kVJ7zx(o)JrS?zKB>t||@D zeBgbVopB;#ax&umSZS)xCuXSI)HhTG6R!eRH?)QacpQ5#6L!rNa(`x=`VUEj)U|nB z1MMG_Tv{ZK#mpijK)fq&ckNP|V4+@K=S)c}ve;M#Pdu?5l^rr)DvUwV0PT?vKYzR% zGPWilY;hyPpFoR|5JP6?I@iC3Vq6S&sN@s)yy2Kk_{_=#E{tj(A~6Gn2o~=^zMyvs zejH=*na5H)n8DO#XSngd{F-OXphTbN9bu!~RA1@WgFi`~<6C$z-&Eg~>%F!po2S1_ ze(jCXcwQ%!S`|5^h}24Cf%DGYlJ8~b8L?zf;0`mM@)Jd|9&jr#{?*Qg1XJuUM}jTV zML9{SGQW{o>!LsKk$gTo3em@>#xK?}8b9NgS$?dN7ub9st#1lf=`*RfERqiz( z%zTB8hI6(Wpm4#3HbZ{z&OHArOIRM>JR?w6>jxW$d~1R( z8=RTg(0-+#XZ>UEu5%s=xiU`S%_}9ZcU{{C`IHp8yqFeq7L^5hHPf(B>{qz0U zx75z&dEB?!YvH!0%yFPn0dnvtlCDFL)%Bh>h0|%OxMnXF0(`E_T1cWldfPUNA#532 zF_UFlhm*4BwrzGZgWp~l89&g1;$Os_(e;Y|xl=2m@`F6(@A7#Zg$6~4{MITfoS(mY z#oK2mo@6)ugHMq+fCN82iP%cl>0rRR$+U-6UX}VIBZ_N3v^l9y2J@~+nXeeKV5tl_ z58#~`c(ljwfpHzaef#fbnkmRlut=er45g1&uFAxlaV4_Qd(S_*vcPY6fo5V{29CqR zh0CQnCWemD$tb;75jw?v?k%iaE$Zb*lYKU|?cRSJjsw=kp)Q^XpVWYrI2cu!TG~H7n=oNXG9I#<8 z2XoyS^Mf6^!*Rvnvc8xyFfpcXmSrE)F%hEOCa_GWBD#KOV3`AJX5v%eZiII@eMG4w zP{6>u6syX2q59xdCM#LN@M@N#|``%$kWIB0~(ROY~Ve=g* zNO-8sq+gRLR{DVwQ!Jfm!U>SpZI$h+6PlG3&djhh9*Vu$hD=4jV#(`EepWBB)od_U z1z*Wewx!;!ADjqaCwDW1G6@8ht6c*A{M}l8%l0jf?jh`J4b);-n=1;fmgB)4p1;ZG zDDk{q6&;eqX;tp_US%-mWh|)q)i{eHZbo|{^0}=bKxC@sGOV$YXz)91vn7~h<-uH& zQb0dByDZJPD`EGPd`kqAvI?*g=B3fqa9H9Rd{L`va?B=t~Y&l0h{I!^E9pG>!S z#>{UpLngb5T`Uqt6sO=~BOjkJh)+u0qiSo-es@5}f!h*a9Gx*&<5{Eoxc-WF!jSyn zM@qOve{Y;Ok^%FZK{2K;y}YNN_;1tethBv;U%(w z%RNe4t*ldJayql#MMurNnNoO;%!n-U0V4mzVpPdGu`LKf+RWv>l>VJ zh|rXJv9Mk&iDk|e!hBRh$KiV}utL&NkptF@GM$|`tR)5FxIigOLHS7vqDnsGiFl7bTk4baLCJDyHe`hWp4JT~ zxRJRy9oc;pw2eW?wv3s^8AsUEk+&zZY`Ez-Lo@iJt=-gFZhS`U&Ct+KB$VGUar1N* z@v1?8ygBYN+o*ZMCgDHM7MC=Korw86(SB>G1fFAvHmj{-oZNU|ZY7bG?7% za!4;s_~l~@pOTy7Zo^+6AY`23W==`h_ME&XEh#dIqn)Ei1rAP5;j0oaGirRuwQysr zBa#0yNX`7Po5nBsn|`gMKsYvFEKdsi0e?F_b6jl8h=+@ms+m|v$is-!NWtw6(@?$V zl_q&yu*vK7NYkl6M5O+M8>hB}h=2U?wrE48%##YSN^?I=0+$V|M7{IRFWf36;()R* zxJPdQDzTQ8c-0|B0$0G*)swoM=@rL%&=A*ZOgwL>7z1a%8 zFKtztnNhe(UFtdIA>1N=eN!pq;(cN?j@4UgtmpU_OVf+Lt5A!~Q-4!7z4rNbGV*<4 z`3S~~rTA$L`Bs@(J%h0xlX-Cme-na$&VA?CWqV?s!6CpeZMEoe$7DyV^%f(Y$CD^& zqb+UVeb3zQ$3puFCqi%M<_{j4`f>6W>Qts%OZ(sH37e1+(`!sDT=vci2*%*lcnLfGx#FXv!uiQm` zC&DPMh8FaCMRu3k7P2;P<>)CU&Sw8mr%`j%w6%l28(zv})E#p^r{~M)l3_X_Eef#9 z!fgwyX5@Oqx9=Waz>)cTxBx#FRZ7Q4&|@q3fbSjP*Pt|Bw)q1)JAG_&4Bc0~QYI5; z9l5@3gJ7IgX2*bCLz?mlb1Z8!pV-p58bZOp4MrH)-?C4BM%`bn_bw_v8c^mNSm=5N}{I(?E;74 zX%b#E#TsuQAAXq1n>W8vD~|I|L(Aqg?g=aXtg!r5BXJq%+P*yi5*0j^`Ml4I6;HT7 z5db0$wG~_=*tJmS#%smF=#xa&&Jz8fS=qB8x{B|9vz!fwmKbQU8&%pTg}ZM=3#kzV z_ZQ6}eE9}~T4%V0Xs%r}Jw9AwZlZ~)%XtE(9Q39 z5S-nO>sGi>EdT88T`M*cJ-QO2)(J{jpdX2j!noU=B@Ze69N9Z*ygRJ((WnKT=0Xa4 z5>HTd{3T)O`V-xs9(FA8^R$B+<_d`Zg!1rg#WK2+HXS(SR!(O)SwKq@O>%tXdp}KT zpzS>sB$N=B!h1`B*_hr3l_}mcGqYM@5PwPL1j^?PC&BQ_KvG0v0}CmL3|yC_fNyLi zaib~0C!;PY#bDnTXvPWs+Y5`ZCeOAdxX zCQNr*a)lN~1JDbninPT|6#xvPr!u6P!D6j#QGyAlSi+iMZzAA8s4!|Oo;I<&P#87f z1}&8+%t~ev%@`NRwfE8lg1+grWmTX#j0Luf0bat{$*Vv6?Oll&1AW4N=p!AztoBEDh8Zbul!(v09dV^(vw_m;E~n7Ix72vc`pWtfDyKs=Ist`7lb zYP5YlV6WodgY`h z&;}e>0a?Pt@c>>_fJG=UQ(rXrUsV^iQy0~j7nOpEOwo~<;9xV3M&qR&z^trFp|Dga z%#afXVTGYE$^|P&Bhs+bBC)Q+6RvGR*Dzw6Fg8?xZ5*HlD1 zp==t)lZj-JiTHwSbr}Zi=tnw-A&Z3toC4Q#(PpeD$iv(YfbFqpp>$-%VOD!U+gMaL z0Fg03#R`b$j_fdp`mKrB7p7qXn6*PHa>q32r&t2sKcoxsl=5LGrqWU=$$(DfX?Z*- zZDL9~XrfbHDB*7s)JG)=$rjZu)RQU*#d&mL*HpM3ux+Bz<4Qp}-b(Vs)G51Y8=Uo+ z7zZlqTu0xvo&(e>I!;k&;b#AbQzV}1(2(z1y>Fk6KE@waF^Kq{d@b-3Ge{J{jt>gwJni6ufU{X-fc+B2-`YjYGsmBSgS6oO)Aq; zI7J~w=8hx-a2*4z3=5D&uDPO|4O?(UBedeq1L}`~nEDmC0d1YYpF1Hr$ZOS9QLtrp z6nW>C@!SbU@@ZZaznY-{-@R|GhS4I()!-?p@Vi*TJjF`oVea-G1XNzd! y-^Vp%pcMc>T*9)K0*lM!C8AZPg+G7PFFQ7O_Sp6RwD_p|> literal 0 HcmV?d00001 diff --git a/rd_ui/app/fonts/glyphicons-halflings-regular.svg b/rd_ui/app/fonts/glyphicons-halflings-regular.svg new file mode 100644 index 000000000..5fee06854 --- /dev/null +++ b/rd_ui/app/fonts/glyphicons-halflings-regular.svg @@ -0,0 +1,228 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rd_ui/app/fonts/glyphicons-halflings-regular.ttf b/rd_ui/app/fonts/glyphicons-halflings-regular.ttf new file mode 100644 index 0000000000000000000000000000000000000000..be784dc1d5bcb92ab155f578f3723524a3dd9688 GIT binary patch literal 29512 zcmd753w%_?**|{foU^;hX0w~U=bqhcl1(6Nvb)J{LP$Waa=$}B<>qo1h^Sl?5fQHy z3@Rvsm7*022$ABYeX&1l3tg19UZPd{Y7=d(ZPnK*Z!eHN`F)=`XUP&m>-+!xexJ{O zH?uQy&YWkSnR(`!XP)Po6M+eWU=cP6lF%}8|&%ddqyBm-N z{Tbxb7T>Ub5&Qa-3;A|IxTbl@!uc_wt`W~KsKouq5?nAIk=G#~L%w9miksK%HQQQ{ zzfTavPj6Ut{ruBkb_@}Og}BCEUNL`N3kwKu2*ToWl=rNhzhYtg&RxKL@zsJLZD?6_ z)6MT)KY6VnEc-dCU%z(Yf<p=6vpVK=EbUm|aev2Sol<97XHI8v zXGLdiXI~kpyFL~$jshU}17x8WWT8XXk=5bpsP3rg7y`(n zIwk?~f{vDsO&zVBtW(#S)#>Rh>8$RIb`I$r)_Ha3q|SMrEuEV>TRR^k$lafGpY2}M zVffuAzdQcBB_By=ogbJ#NcZG;vOPAB$)oq^in@!GqD0Z(i~d^lRneb|eqZ!a(Je(c z7p*8-T(qcYUeVm5=AxNJ(~Bk+jV>Bi)L0ZPiWI)7_7<@IzyG1}62u2Jz_o}yTA=aj zhtMB^C}pn}Kx-Z(Js2;+fVfHxf(`LpH3)XZht(iB1fdxBC(c1#}I^JNDoFl zLJb1)9itFNdk&aVx@ONUs!x zPPD6&a9)ELICrKYjb}Qu5OR>d9kB-ixC{3pEezwwFAxLw z&Rt0VQV>2yL_q+xojbvUAiRb6BoBh{HsUip2*Nvvf5n3!v?KmI4}$Qn!2a9DgCM+z z*ujG!{06a$2SIoraVZai@Bv~!4+1!nz(8B*M*d+UA_}P=+@vm6KQemx|IZ&{%9ngF z6Ta1luR8(*pAzxKdcc-Q9yHt_1fFL?)u3YrS@cW)NIdu6+TkMQK-BSSzbUXicV+ z7LJQfeo#IlfbN;MP!5Nh#M-dlp!XH~1I+J>hHIkui9{peklW?<)dWOeu~{^D4PL#| zD|wXm^y>OyVQ0aZap5CH^Ox`c<=T>=rVnB_>dwaQEggHy@vmD3>0bzs8&jBFKYXyA z-4;{Y^=v0QH|FM{{VloGGiwhoyXCuqL+fHywXyxPx4yD?S+u!2$5A=EDHezTzc_1^ z$B8G1@Tg7lxULP-7V(4vy6^s)Rm!i)R}n9>dqa`hnlfLpA;5gadZ)u}W=@CenE2(o zg9q0IDl1=D`S|^^4>Hy=gPFMtS+t4OT5HM-I`k92rd^Ug8!~3%Oq=!oi6f_)jfpIynerv~O}wgE zdN%R*EO+keNVFoyJvl1fXv~m)D%p*RiPr3#)hjD9neu_m!lbUMtEAt2Y*Aj8D_t8ZI( zOLJt{`Yi{Vn)Yv5Kdf%{+O_MY7e-ty516`UNd5XvcO08O{n#Cw*4GbNGj)JG8eJ@Q zzbuTBcc6cbBu_DWIP5GH!@THQWpxD<2Gj#x+Ol-P&stk*TFHxBwc zkvJeWBhj@X7L&I0#BsWw7=GzRdEABL@;Hz!%_2nV2boGO$>*rR`I`keR*_V}tZ1jV zxD1pW3422>U9bGVy??I2skAr?3Y@IfSs*s2<`M@|bC=$eb9TLQ$KZ#x_MPtP==*wV`EOH3 z&P~?T11}||T=Rc&Tiu<}Jh`;r`|NR|C7MA*OAN~iMnsRfH?*pM8{gs&flJGQr>@Q4eq1ZnwMC4)3ed| zy64ZIe|{ar5b(>Gz(DuUU*zvXsm~f_TF@bu+v0Jhy(ggfg-Il*vU9i&7^09XY-!SfL3is01oMw=+<0u`OONSvkBOPN(&Wm24|CRYu-M^_clmsRI@E6Vi2O5HsTfyq*CrnqKf^Q?^^DGDyGgj_z>R@RGLqE=-UPD8ENsq-cmp9W_2*&+8QgS3U&jTUppg-(K4_w-?!PX4|`0`BFKde7Se8I9ECN%{OeuH_8Iw7?TfQyu)l%()Epc{}6<1$YOh- z|8f9Vl1~KYle{b};mf=k$cS%!U7q*@JNlM$pW{t-H1TOD?_eIam4tLw3GwF~1Y!^} z-^pU_O~Rp$VzfUCGm>aX_+WolK8mx-xbhLZ_2^Lo!uLz(6ceySkD<-zYsi{Mfr(ov z#FbE?s7~UVCf3vF3;+(ZkIsFxckbN1S|p0f;jh1D)4o>XJI|lr8JCY^h ztaba7r!;0sJXLH4rvy)(Om}Y87%d{sy9Lg>vji`oM*&dp^kGAR3ZmE#f(J%w!x(w& zkquVy#3L>DK7W2E@!(TWZciMzBrACynRNbns`l3H*oC+BGYd$1gSCkjicJg;Nn6Tq+tPaP&9fbY?p?QG^)g^U)lME^EH5{Xn5>uv zRcCthbQ3u};0JAd480i?u0oGmp+&$LC09d8?@i28h<&IgX@UAk7AC2l%fh|#a@+M! zfArZ$PhSrfnPJ}gd#3;WR-WwYFs1EHGw~m>xhIYNTjk9tkH>CS+BsXRyyLCatKYhV z=iXOp=plB7epAvwo90GbZk9fS%miMU!@N3cCWFcb`Wh%}qHdb5;Ezvj9kn(22c<|0 z=1V-Dyns6Zqr#F}I4tlo4og=W#e!(?V?L;mSnG&Y%ZANJ!lZJ0`6o$%5A z6$~H5XaXsLdWjWxZQz|tiVbWb#S^g@zi}?kx0O^PaR5sksL{h8B#Osc6^pS-6y!1t z-KG_c0I5_?WXjWVB77`C0E0X9N$$~z7hXOe1-sAMkd&T~4x>?4OukyeKg!$Ss|6H5 zgB~bOk%}NSOT8$!b!AJRrG^W~W3lvW_(!D??CLo`Fkp;@bdj&gQl!RTR&3Ba+^!HQ zcM>BYMw~rfP*6Cvkbcl06VyMyHCmL{3Z@kl7Saz|0P59!h_)Coo>-$bXk4NXvs9SR z6HF}jXQj^+Q;59=KB5$x&J7=^@jchhecIDX(a}&ek zaq&bvo@jmCXf_+^N9}Lu{ej0(tmnmo;H@o#*0YK+AJaokW}(q74zR({(gF=9v%Bqb zTXDIqP_I|+xK6n-JKxmLVqq&Pno8`~vU{gw^{-X79}C<(l=ZU*%$d@sUAF2xQ?9`< zbf_y*`R9)Y%p5AFv(pbMKjVFXev^KNx?$@i#U6B+n8{|*!U|=?=#N^iqzg!Xot4&{ znled^`m-4O&AK1Ey~P=(w7d~D{ntD@Q886Ci0Q79B3AjGaW@>;{k>V6ZlCj%e6;Ps z=ylQZG=pRcU$tiBwC&?(8N%gKL%zEp(_#oIci%RC%KWbF^QX0NGgLlcYIBh)+oT4{yo9ax;B(`_Zh3EE_-KeH0}s1>WWM1zi|8vM8yb;}!f zhO(RiZ!uU31~)ERJQg?5Gr9D$Xe*Xm5Hp*qC}v^p;w z*N{S;G6K<5kG?@5T>?=z=@LN2k=}Xf-`uBNVd4PSA2h4_n67NfNuN0j;swsG4xaJg z7L*Pbj#Ew^=PZz3RJW3j!b0VUbGT$csKSDU|GP+LcF9pJrBsJ=9lH5vrwS)Ti|K!5=NyGy*{4rGE8dDr?fg=uqmT+G`HiEHcE>4gPhlm$92*;Zd%Ul{ zpmt$35ulqOKA6%j;t{EBA`5A6KB6PRvexkL+I708Ne}>H@zhp9`it*R{N>86N@>x- z3&+I=F1F%dHA>wNv_XcqkjF)D`$D=XZK*6u*orDEi^MOB_}+k3N>3)%@GB4CHv#nt z?eKeKAnG4CEE<Mp%Hx^%i-A(-muYYU(^2Z)~Z|7t3D;wYa+m6+L8#*+-c=@Wm zW509ThTq(o7(us|Eq@Gk^yo;icf3SH!mP#63-wZru;#W47kX(!x~`LE(6$}Vi^47N zi~60;0vj61428fB)@M?iHc3)I^p`;w$?chLv7dAF#F^sX6=eK$oe@it)27o_nti2wO;QUQ$BiYO?c(b z$y08CxwPs&TMntO#Z)Evb|%dVLKxVcG&vO(48(u&^5bWy0(G0UOiUy_ndu-2YWw~_EjnngQRBr9$MJm7l7k%1~8!AYCYpA$= zT8QnrQCZI0jvv?|#|imD02riJ?se-8q?N#qnQE_vj^0^p))|_lA|{W!SiMfXd;0cd z^)uNLWtSoQ>R~g6)n^ngUOcz3fSs&O;xNh6oW$WSsNtI47tQYQuoc6~YGD7wM5eJI zeD(vM0&uBb_>k(Q2OsnXw=bliQaNbYG3DtbF3J~TOsU_U;tY z<)?53WlkyY6HG4WZb4hH%kt7RPE|NKt$?YRQdX67>@#HyaYvH4pnf0A{>X7t(qyZ__dbhJ@DNS8g3wYhwr*rrmI;~1cYLv&N zili4|Knm6RtQ`GL?L(L0OWR9m5@8WgvY|ynH;~r?jS)Uvj;65>V{deEnD}#ewk9Iy zCf9fBXLQlI0$x2AkJ*d7qcy02{DKo|6UG&+pQ&SiIoz6vG^GdTW$-wL91iKx7v;xf`du&bMkZ0 zDWdmMHLyAu+rpSOw8C-)tR1@fFQA+MV((ry8G4I&Tz;T0q~q_+N!MMs!}?LK-r=mm?8D1TwQF%q;k^xz(Wtad5na1(q_0unK2 zkStczCfz_zWDaN)WH<4v-qlWy>udvx^L@eL!MvsSw8|EPUet-{vRSrEc2}BPXYm(g zv&%;%@khy65o!*F$CYR6Tka6`CZj9kVuwa~skwI_5y2mv$! z-JPnCPwkP(WTGLx++|&IKk2l%j*I$4T^mSmmP?up==#je0EHj9kky8pq-br}Stz=7 z&PWt_T*W<`T`RY}k@M25_=EQqzV@1>--zX-JXZOU(U)SQmzEE*jjyE6N& zx3gD`g#u^M0q@C^d5_&5A2e%fG&3G|OuB1C{8!cAjgMLGKJ!NQ@~h*cS7iSRZSJu_ z*h#iZZFAC8V@Xlu@NclqH;?>(4VU1(nZoUN}no& zm0_%$RVIri4)D5v!PgFGvP-RS2?GsUQT^PuXEyuvBk%v?9m|r}*nI83TRc0zJo0Si?GC#&vwQ=pj z{(yY4dP&pJ#?dy)Z7*cxo|-))T{LB}?+ui*oxgTu%L8SfBjWJcz}k0RyiJ}3 zi9fP{qoBZ{yp7*GW3&qKHMb2i?*RCJMWOK*m~Rk+iJu%R;mBt|lIY3;x!b|l66o`x z`45*y3ngC#D~3c4n^lEKl(9+_i!&Pio`U~!+3e0Qy#@Y8qfZo9k%k;xMd|;#&g`*? ziGM18l!|S({bY9KbkrhkVMa&VVSlx?HPe-CYPAK*o=JZH`+*V;C0TDDYsM1yCu58e|qLKI0(-%dwMusZ?{BW7uS~!p1WyU$dRrq$O+%%@ti!fDs$>k;3swe zOt@YCLJng`F_`?_nZc|t4(Q-K(WDO*>fA!8NseMOmUNMb>J5dmojfPNFy$|D_4y+w z-n8bC)<@RdG;w6UKDYOU#E4C6r_8FnI)g#>?)Vygkk?ECJTFS%MHY_o-(WN5>=8Ty|-h$Id&pc$D*Epw+{chQY zVN0{;l?XE0BA_j8*p~%_Iwt+j4c|pi=htTtn&Xg^!Fba}B5}uC`aP`ThOF?hIrm0;S6zLX+Np z0?ny%7Y?+LA@d>U!o}(U7{rfO#X6ylmv_je&z+2lizmuw_4`LL_<14{$byGpU)@TQACXCAB4nM?DW ziH(jrM`EKhPs)lb``Ih(6=gq`!ciXC3xQYiu;mt4wpG~`%eBw>XpTKMrtGq2yDV&Z z^M+>e7s`K_gN_PErsFZ;;`~2 zxwpvUkUoIjF*>TDLTs)8#{sSoT)4jm+2IDD18GGdc8~qP4wI&ldEw*jB7dYNy}zcB zsYX6>3}==4Z2$O$Prmx(!twrWJ+jv6{@T)piXv+Uq$4mEGyt`DGy|H?+ zGWgPESV)nOk97V1H|+LPtUv4j&!6MB@(p(9Z{Us93WF!S2mZkFuxREfe*o?xJe82Hr(qPEN8kx^iW9sEp$L7-p|E;n{Bi2 zvy#pyDGQF%e0CsNhBZGa_()+(I@b@B`Xs+6I7`zaOxE6$NHT* zrMyS70w-*kkEuph1({|uFApmalndC(z?%Yh)sn30QSn=)9wlT9|C z7p2S$i#{I84rOMZ7Y$Aq8qVMy;FR~sdx&Q;gCBc0e918)>Lw2fe-y3~?3Do>6aMtW zAO2}V$AI0tk^b}X{UV7&Bo#vg zBX?XFBhgMM!+9hbyiUpI_gM!s_^O2AlM~9THqYDch&A4pbv{t~WkI7~c{#t)599Uu z_wI}BjD=tjmfOnnPyIZ%RB0I-t7pwc{bQAr*BEwIPFB9?yj{6J#@4pK3+4xbmE)uG zG_n(ezP#vpcsoK9*ucoN;kIkT&Ld86et47m;G~ zADaJ({++k8wK3)X_IEjdOamWr%G1$5johcE6eLl^xF-lmP-O#TQRiMXI9BBL+MBqb z$ZZAvL{;fK7~&{RjvLrAbB5Kl!kjUk1*R`wF>U!~L!L!BWOz2;JTS&e@6zX4-pI1q zvXm&xkkciDEQ>nhBQvN0($Y`$rWUiqW?nz8b%OGo%fByE%(RvouU67$v8m4TLZ_pE zF;UVF-)LZRHKriVX9L%&d%Swi|U!2ZYn*45pNP zL?u}1GUcH7DWu^^pURnjYvSw7@0B~*)CsNQ*!rw2XXcHjXI{>*WTXRS5vL|99LjUE z*x$ZT5toGdv^MF?kTd!IpS*khFnN*g-0ClbWK2@INQzm5SAyFsgwR2B+9pE8;d1M8 zh{4F?%ALw{sB*of)ZF6A;+Tk;nfqQ*(m$X2k}F58JQO0#uwVLs&Cpu6e7f@XG!x5Q z=_*oo==9IZXyW$4b>R zK%~1PJAV=663FfjXf0})6$gWek%4{&k+fC@pI)4R36hHqo9d|8mznqmV{H7?;%dn( zv#e+1TPJ{}9(I(6LXttB?Rt6Y7wqryq@0Gv%w!qVgd0{)1GKZ7 z_4$_9T{fGG#WM_9X;P-`;Tdcyts_`V!2=G#PZjG53ne{FiM!b$u0V$)UbF9_2Iup= zbN7CD3uo@^VP&O!Xs`0Qrq;6WyY<7pa~0d^*H{_rcX5q61lU=ebHS6->EQ0G1RP=z zB%@k!Iz5$y0^rK$*tG_51ndwpx9;N_GZl2=IpyqYr%$Hf+!tJle5AradOe3rN;i)5 z3sA3J0V)?#mt-~7zm@ZnWItyK_X)eGr!VOZc!5AX zg{27FCGFSYGQfHS@vBgby7Y+QtwLlj(oO|`bV5)M+YIS{A`qgHjz(x3P{@jKyaIQk z*ou`!NkJBcdrQPml!uajy#dxoH!fl8<_a}k-d7J>`sX&KSsE=)7=Yke64a&T>5G}k zm7SJ7&DB(2kQR{o4bU^)qP2y^KFJ)&G>^2VH+lkDp)8r{D`YV(C)aJaXXvx^<#~Ej zx!G)&k^nocByC=)a(kt^zOj537v}RzN(0lyn zm~46@Lq8e(mJGL{_(r#PZGQU5oD92cDom>?lx<@iqp(3Vn#9!wB~3+;4-HuvOw7pe zxy33mGfi@p*$Q$B@(Z){j2VpfQtV1cJKg<_=6;TxbemmD&v5&l9z%tcDe2@ApUWgI zu?79IsFzJ?rV@kEL@G|wo(S_WXAWyNSHHT0Cn>zQRC1Z5LK}eI<#0_C*SWMJTQQyC z!A1g#c7c@cy)S`i<-@6R41~5Gq2`hd@a6vKnygO}8+fA|y9EOoG_pf5#O%XL4JnBn zv9VgF$X}#eaexcMI)~%4R_vPmvX|DntAJ1@LNTAcW{f$II_`Jn^y0m!pXaL+nns4xzAU+VF$c{P{P+RK+NU6f1Q zYTj>1Zt8K8Rx46lQ$qe;yfiyTuJ3&~$tT`*c|0z+$HN>f-Q%W=*%GyeuMSrf{Vh;L zx0K?5hwjJ+F7u>UJ*FS<1U%kK?=)sMySzvnx4Q~T!r>B6P-iYupXF6RtPzDtLPY+V z+ziQ$I9CgF&z+ETryz}H; zf!Q~V8hPq=_Nu9AWOM$gc~cG@nYds?-i)i7T(ehQ%ju-P`)hfv{1f0tyB*jFpuh$5 zp`)yHz!ryp8E|pKXD}R!!od;O{028Pt!Rb;ci4a0m$tLJ|323iC@Szphi)Bu-P|F{ zABGNX=P8yqbm&%-VQIT^8x<*t4rM#7{DFD4Ky86#p47VSCsL~NkC z4~9!UBu?cAGa4IbG{&SKIYWWM!a&H`HHx+i&%p%~*BfU5JamLMh&7!;6|{6$p+~H4 zavao?;+=cyg~3X#etsC1aSgoe_63*(XKsubddY1ipF;7(km5m;qUFbS#~zWwf7D)OqeL!D+ezfdi7Z40<)zxj4r6mcIpk{o62e1-9tt} zB8dr$q(@<+x|&9l-05kR0ZlG1f2BXEQl=*PNoBQy&IMT7t#iJg+?&i z(t=RMM1Mc`+ado9cXm|oG+Is8^lDSdhtFm^jOkL7GFTnT=$7+u)z>^NLg8)mK8%_{Gm zf;s@Z#nbp>mDk6vhh+wK8&%IimTZ`C&f!uE)Kc8(`I7pwpu^+dugUt7Rn)3=K$(lf zdF0|;>r1KcVl}7-U>Bkeu2+FIo;I%Ju?dw0s-{yRGVdEYf1}6F-i8`s-BvpWt+D#t zR0VJ0#g5|Ur8t_Tb(RON;aCI67!~gYk6LgM-bF|fhpfSq$HWNMLO{LP`6?`cR7^B} zd<^)WQx6RpjY0}kz=FHGHyJKs3EyK<5~!z^xdECFEi6?WTl)RCumKkisA@nxNsNyW zI1MmWL5>YXHoakka%evSoe9|q1co&{$z^EIp-ZvMBVR^_mwjJ;@ig~P5o=Yq6LL?1 zCQiHheFmo#EYm&rs0z{__S6IVgsz|OF0s+!HA=l|(pgJMANTYZU+yD-f4Qm$UV}1< zjfa0s<#&Sy-3p1+Yu9l#wWLEQgB?F05TAd9L z3Q0E6h@%nayB*5GciH?M?A)4@6%t1Cw3@Ly~}3oNPOqEN2!mgKX09o z^rl*X_FZaMCdVP5k^Uz1xEvj(Wj!J7I_e4Pm@+m`xn2+|vVA`Fx$sPZ5@$yKNm@kF1+Q4>cU8pW*FUVaEn&urJfoWAG`zW{W}K_ z-jV$4RjKmL;)CqrcvoTa{-z%sBvMgnn)JoAYWLMn>PW1uszin{GxgL8Q3XN)_ZzIl z2J@0u@{S}!042UvJ>adVM-|<~*~-eEdbA^91dG(Zm)5f~{*+94mJkr zP3Y@1&u=m5@`+jCgfS)cOa%@xg94;2yvm)i#9400DMNMCN2D8A1eiyVBKbx=*9VFq z17HP%hfbI|k=W>fc*`&gcU~^*NL{0?m$7`>k9pgW8TS>0+c}^+N&oFY&L^^K6 z6R}W;|H)H|?ABYdMieQ#3TnOCdYy6;O3RNxUV1~hirUTo*BgW+jhp&QeULn>HZEyL zp_Ry)ob6#s7fK{ws7JqmmzOqd5VeZ~k~|J}5*Q0|6jRPvoG~Yh39dk0pTo}OjKzzp z=*lu_ohyflb#lW*L}&$>;Yv>^0GEAs$7+{CzW!GhaczY+)f;$ zB>i%#oI?YzD|PDd?xzY^e^AWtjfzjhHo)B~{7VxDu)MYN6$~#Lpac6j7D?VYEzl!V z`lrmV%+$)0`7OR+0md&WSl~giAnv>S>AM%i7bx%HHu^0~$dbP+KSkCqyFriLW1$p= z%8r~t&{<{JVPnrmP9i_t$5>I*!;2Qb_1JAiMNenx?XTKvverJdVdKIzR=xQ<<^l5d zeHs1lf2e)Y;)ff(Y@fBte4kmiu35ZcII9_)YY-LSb zc>*1?!t5+`(4i!}f@6i~Dx1wx~S9Nu`hxbm1Cn_4qy3FNC?n9%a_bu>#r&YX&zx{%*L`kWNWPLi`2`d}6 ziJYg_dSOALOWv33L#8Ia+=B-ETvGcZkFRRP5H8BK z$=)FEN$LbO?z0!D5BNIMyJqwNRjIZ=)~ileQWm(Z&P)~_01CgXze!IDXw;RxYhvei z;sg4;w14UJ37x_1qh%5ppdH?WL|L$T>WOprQ70_#vCS2c`m)XJ+~%_SNX6#fRZ}Br z&6~D)#*EF=XpUTpLlMq*z&EBZ98zhG?Dl+h{GQ>}g11{k04f}c%@ngcGopd#q;X!9C z=q+q19yF>PNIn#(8&i)IL8S;*AH6}zixiGH)70V8;Nl(-MZ!j48?QFs0}R3Q>`Gcno>A@aRC*P*9qwX?+$2H zzCK8QkWG2~HKZCgXDkQK#w$Oh8@mU<5sP50$3R8p-85g}!p8du_BtRBbuBjsxSXn4 zz~zRvmXz^UgI7Eeh>Tg99%{I4R_-HnZhl%cr;k}$UnMUcQ&)+q2EgjLbWC=UXHnzq zyY#beeEMcNOA?okscm*OoVdj+B*} zHlUGVD@=kA=?}^C2(Ci3JklEhR6CaR83ZQU1z;&u4OL)hD1(A{Ar3W~@5`*HQ{@io z+Y!k-wqQ-ztp2fffAUUXR6L7+JC-6O9jUlT#Eib#fUdyQOpcGB$RqCK4?!3!0L zvt0b^>PX4pYVSPX6%efxpoES5fy6IS?q7V+Y{uJ8ay)k6^d?V(z8J4ZfSnCTQ2bt) ze`;XQlI~%77K^!`xkUL>`4z$t?|~@xW1{msi_%ef{F&bFrv0U3OF6A!3n}X z7$wTIDjig)3HXQzD$VC`nTJc8J#tS2$Q+Xm`zE}VNE14xEqvy5ZJ@eiYo@TuDQmFE zRq}0{=n5@ONV7dcvxXS!Dn<7&P%Z3k*5`$ zUt!j=3&rpmfcJo0W_9G{+FVl-=l?ozpe;AgVO=xWa_dx^-sYI&!0*&sErXShZU~y{ zM%HD};WkIPAw54(f!FR-z$NZEHfsDvhsU1lw3piN7_a8}qqHqs#$vf*LgKabtA z0B)b$g~i!x>^1d-8#|$lkT=p?LOU4V&h)2vt!~6 ztFFjpOt(l1`o`_H(X{!td&#HqS)X1~Q_0^&EOhP;}*a(7OaYz&N_ z;R&omD8Wn;RVn4 ze6S;}Xwi!OoCk>T)4H4MAEPdKbKrHp*!R^$85}txZk=@eLgq8KZB87v^tY_CSj1-U zgn7?wQxcMK@-9Nb>VIds!$aXej}+OU;W9 z(vu)>EoR36awH!8KnqVJPxJ9=HKu!bmY#<;2G(Z|r~4atAtd3Gz6)=MrZU|xtKs6k zWEqMJ5SD3Wsl4`#kc%|Ihg8jD88G%BP0!FZR;9W9xL!5!)n75hBJoqY1L`B zrtM1?(#z6Erf*39hq2B$$M~@Eu<@&mK*qX^XEQoXxu!Lyw=)Bo_n1TG?^@C<0m~xG zz{3ATeWSt?ONM?w!^lM>_+% zbmTfFIqq|O*Kyntcl@X0AI^MdlXIQ(Jy)6QLDxBViF=Xz3HOO?A={B%o;@l1iR_oN z&t`v}W6T+v)0%T4SI!-mdnC`87t8xe-skz*`NQ*97c>_fD|o$7EL>N3swlr`LeUYA z%TwdI!SjsgjOTCO67Ll6J>H*q|5jXGJg4~a;xoQ9-w@w2-=n@0zRyeYOClxnN_LjC zm!_2tDqU2%r}Q(ND%nzY!k_OS?qBCWQ7)7ZEWe@rNcqqv_{SprSmSGU=(9=c zWimXY@LpbJe3qJtrOO8Mq-(Ua9cl80rZRECB_?q=EmVsSuU)$~fd9kP@0DAH|KKs7mtT(l z@W8L-27Em!5N_hRg~Cn3LR?*g-xx}cLd$1iUS2JXMy(Tt3BpvAyBe@=5EdaU1^mT$ zW(vwL##<$B;I#ztWHra7L70x(XX3erK4D!BX+SSn-xdQ;ujgj)cH9IESMfeb#c2|6 zg^FPhrb|%rX5o5XehpfwJ`sSgUp25_ftD=?Oe(Vo?W49YK#vE6S{~}q?;-H7zVQ9` zt?YZG`o6kWpl<;EeFH|h1>?U|!}=y%CHzKbHjzzYli3tDl}%&Q*$g(5HM3c4HoJyh%dTT{*jzRb=DY>$db~z%AzQ>2 zvn6aPTgH~-9KZ^;lC5Gb>_)bl-NbHYx3D#AEnCOdvs>A1Yy-QUZDe<_P3%s#ncc;< zu)Enk>|S;syPrM4zQZ15TiG`D5Nt-<*~9D+_9)wdfA;Yhdz|gUy0e?@VNbH}vZvTy z_C2eZR~ldb$-Z>vlpOSdWpTve#Cyv{)3%> zmHQ|7M+>jApF#@%8T&aq$xg9fusA!-UT1HxGwhe_SM1kV;of3zvv*iKdzZb(exv7X zDX2yv!!0Y9R##tDO>wBYIvEGGJim|YVJ%;y#kE=-(c-8U*J*LR7GI^tp^<7_J5nBT z%j#7;6RB1!iB_wHqt(372n`9u{61oi1Y(W^VqQ67UO8f3IbvQpVh(Rab&xj(u?8oo z!3k<`g1j-fufYpy@PZn=paw6f!3$~dLK?h~1}~(+3u*8|8a$kMK&OtV4r%a08oZDO zFRZ}}Yw&QagO?9$aKaj#um&fr!3k?{!Wx_!4Ni>)r$&QQqv2Jf!Ku-nuhE{b(Vnl> zp0CxOuhpKf)t<-ei8)@i8k|}UpIQxGtp=}FgBQ`@MKm}O4NgRZ6Vc#AG&m6rPDFzf z(cnZiI8hC+s0J^p!Ha6}q8hxY1~00?i)!$q8oW9UUY!Q7PJ>sc!K>5Y)oJkRG(REOx>!3#0L5;418eIo9x(;e|9n|PLsL^#$qwAnX*FlZ0gBm>tHF^$e^c>Xa zIjGTdP^0IdM$bWwo`V`a2g7QA1U0%2YIGgc=sBp-b5Nt>phm|*jedhQYCi@wIu2^| z8`S7GsL^jwqu-!Lzd?lBXP@~_VM!&&`I<7&Dj)NK<2Q@kl zYIGdb=s2j+aZsb<(Q#0tzL5+@s8XX5UIu2@d z9MtGIsL^pyqvN1P$3cybgBl$NH98JzbR5*^IH=KaP^06ZM#n*oj)NK<2b1($ug-@c z-fc?!0jq@mmf*;mp~HAItX7S*+z6f<8KtN;7*eAeHHz>k#2=^)MM>6RliwO!E(re{ DlhOCh literal 0 HcmV?d00001 diff --git a/rd_ui/app/fonts/glyphicons-halflings-regular.woff b/rd_ui/app/fonts/glyphicons-halflings-regular.woff new file mode 100644 index 0000000000000000000000000000000000000000..2cc3e4852a5a42e6aadd6284e067b66e14a57bc7 GIT binary patch literal 16448 zcmbXJW03CL7d?tTjor45-QI26wzb=~ZQHhO@3w8*w(ZmJ@BZ(tbF0p$la(=N#>kvm zE2(5vQkCfPhySAC*&%gOhXNAMqjXaM8ZdR9h1n(j|bAOHa3xsaUpVQb^?bFN$mKV0Ewcy3Du z@-8k$`ak32WBbVi`wx;7^0Pnwe^+&aJAe9T8!-8dp8P-m^j_k+W}s`RtGffD4+(~# ztFH^%r@=P?d_)fbz?K5R0s#N*H#RfO?CBZn>6_?x^z-v0gc4w+(WBE}13CaHLhywQ z!#%^j8s6#2z4_*~82qM%VW?EZaP{qr6q7)~zyRXUfu8*DIFkvyQi}2zgVP1nasq{A zzK$~<^8~1Leh9gA7?OYdWb(rhHBCeLF_~b@=XwJtb#c@X=&{tLR~#2+TS{-c`vBYE zGBWX|sg2q1)>^5WQl6tV-S^gSSDaqgl)f0g5bP3XzB_opq(U*a%n-{&Nsp#<PXeb*#gCojQ<~*y?%~jIH!wY%g9nHSRoaSF?Kj+nhFb0uC&n_VOmpd_OBYox zmnx5#Y6>`tg|imfwPr|~9o*VGw6l}bCod<5GtgOopG#Z3FYU1yX;{uJt(#*r8r_e7 zFtr;Gdot=wqBrPOr&Auqx9S#4&q}4+IV@$;lS%g;OwuPXe}-tkmpsZwyFbf2RoE|~ z^I*n!=-?L4caqmD0 ze6gB6sXkw{<`|Cx?yb^4okCyXCb!Pswu?l=&V6!>eVjh=XD+I%?*-Gd7M;9>8h)~6 z&0J!HkB*tz&l&C|b)oTW*SdHifwpF*1$>(yA`o_PKmUNb%3cQp@DV=5e(dQG!VdB# z4zOo2dD*d^}VrwZDE>cjbvV3uXQpX;>NPr?6LUB>JyOhwrqV5Mj1Q8A=HxZxa- zQwXEXE4&D0kFPJik^cKOC{0^_Gd~wNu89<_dGZ;!WUzzZ3ld}@(h^<$4X6-4pZP0> z4cT8q?NQVurwRI1@u5c=cK!0A)|eeN43pohgBKnf%Zphd-bWZGHIQE~`m`*h=F^&l ziYiYp2Bli;gaHnZjhfJboUR`tiB7foe6NfemF%KO8OT@`0*rjk^<*{<(SKi84B6$c zSAeZ)XeDt@7mIt)7s!bPz7`HP9ftqc{+RVQxN1rHewmj8Yp3IVyy5+hfQzfO*PnR6 zhtk{-Yu&KlSEH<_;xUIck%#8F?#Q96cq(tN&Y&yCP>~SwZF+9EW+Z}7E5H4?%I{Wg z(N$R$e70H+BskvgkMrx=s0NkTo4j@vUJI?-vt>?b>ZKxs;_5=f0G)6f@U^u0(`_>iKBH|X`>9ka9q#!rMTZ#DaG+DNj4Hb@5WUDRx;OQyC`$YMi^IjCMmr8 zI(s_$k$_>i*!Zw?b0n%}L?TE;8iYNv&D5Okc@@2k64bhgEg9atc=7JTCCwE4`m2d) zotf55o`s|4kAD`L4d20r!>w61;4e~qalSSgRUGOBHl z9RTUz=#A|RA)-_XJ;fPvhjE(w=K~z`rx{{e9EixI()Jy>7>q7pDk!X2)o;7@b}3Yu z9i|Jv^->~KNaK}*?iz`k`wWk?k2H%PP(=B6#}1W+=RSZgxN>tnUk$!WK4gXlQ5YlR zTsK(s$>9-qC_*h|B?@VYC<>v5_KI>C2z_VFA`o{64(?4{0alZ{Nw|H`!{CqynYP_3XpLG_k ziP$}NfO!Bc1h;p(xMku(+}e9AFC+)*b7-cf-zFY{y5q^zfrbBu7o09H&lgsnQ0~~g zy2GlijEBH%4KeBzhNc5k{iK+Y1-<2Q>UF|@>0Y(&Q0+KPt-?=>*O;tSLw&e#b>>(F zM@%`Dp)}XMSMJ?EoMgkl7E2Dlkm_n=3YT5*wm_QDoZ>7lvtsY4O)?QU&&U>WL1boz zQpm^5oPSA<)4GyW3E#Ps%#pgS9&NNgd{L&{3U4mAPIsPKsgeU0qP%W$`ZjtthBo>w z{j$ZZ`}y)?bf|%(x(~j-JG@sY%R;$v#5BH_v+zHz7j`4+RX_0>ExySHVGK_8?ls$< zCG8GiJ4!l$_CUvA=~B4lvLPO5zU!YI$VaRmBu-~t`|-fjE8m|b--_hjHI@%Obfn<5 zqFvMMzZAUzVr-;8sF5B#27-ldl$|mdx)l)mQQFu2FIOtOc7Gu;oB3aT zkoEXW@GtHDhHTLayMa&3)3q|?*fC_}cttu?Q9^2h4(mFdWi>)r&@Pv28u{R72XTH0 zZRuM=#0U~(p`Qab%BV&JME9I}R{we>pw1JgB;y5-iwrmRLHP%hMOR#-7%AknieOMN zo?28Tc1wE+o31Am+Nv4Dye*YinTqC2UW;J%&TbQ$KFih z&(4l%v^}kxB%IPw1bwe_&i`(w`EDZ;rR4y4yR?*>qOb6Ki?AP+?18T2(HMlK=(_{9 zdm{~sd*AEH(5!TkVTELf1xG!^WBK_T~kY*#Ba=bK-yDs2kr{xCsRh;tzmzhb6>9 z!z+!FI)u7k9fl1aR<{6Rb(#qU59Ak=h_2T0ar}&kf$rP4^hRW*)_l%I!1KROf`P)) z2MGiZQI*|?s^T!TAY`p_e+dw98bH9&ELHjiE7;c;&=hB;DbKUs*7chHcwS>>?5k2X zp7QG43(FDIEQzG>$ws8!ZtSL+a~6-GO3XhBmGXD*rd@xN*P6&K%~IvQsKK~mQb@B& znOIXfL%=A0T}>ki50;ffb)L6t)Hpo7O2uKpP*QnuNkvcZ7+jf1M9EJKck{Er0rd+S z=^O6^6DG2}`u2S{E__E%YL(>)Yet6OO*dmT3ItOyJl?OsHTW3*HpI6^v($s$sAGQW&Iq+~bF@Em2$N)h_?PSD zFNSos=ZjgM*=UQLi`D+ET-=unMuvArE5e=BJ$R=i1hS?y}#89}ucRG*1PD=%dmAiyfM#)nR(>UJ0wzQnF2;OY3FpZoVXs+cy2w5;?GQ$<2e zu|#iFD=ow}--1<8ZyobjRWkurqBk9Rt{?GAKrI;Q9zBLzZJaQ;ho{E4;I!6;pT$iX zS#$C8bIak_Kk3dF92Spdm6>ggwrk&Z%+#hbn9KM1UQBdba`4JOzLqFGQ$(Mc6`_Sa z>2U(>7)j=}3e*Pz?%(KIyA1H%1{)%%Nf*%@0bM+D+(`kq2KwZ*I4VfHF!=@9FDvf( z`D5Cx&Iap(E)z~MuBMM|Ns<5%P%f*;vidnD<8)(8dNv&jv|>5$nb&i>+#`geKYw6} zs3PT6u=@HGWyd^;J@9Q$(ot!|lp4;Qrkl549^Q|)eBMOVeorn*`w#^4TIQ!@;j7&} z9jKr9SzUF3jZ=DpFN7>#&2XI5qjeoeB~fm-glu&dEb0p1Vc|JcV|rPadNR7eIg+YT zLWliky9=Z8uLXGp{|#G$P#Gg@h1E>)KAdDmO{b&8e2ke8G}t7k_78@NFc#F0JXn|K zBvx!abv-#UJu8Tw>T4$Mnk!cA>%@Qq*QbZ};0q`@1DY5aSuFp7Bp-&rG7uC;x6rA7 z-&=2G!#I_&T8pGOhQO5XUKHg8{w~_v^~rQ=q+?je+e{P>8?c)n&tiGj12TFTV;$st z=imv0loSAktP4ipl*=6htfl+=WF}G)C<@j{hH6KSSnUA^irkKXuN>mhbMO<&)L9qz ztxRgH)b)$4gWy-G7G{hdY%H>OqmH8Kiy4|O$&Qj{IOnqbUcP|=?pi__3Uy1aLIaXT z;d4MJh&5FK?Qa(sU1p@pZKR<{N-QlW{S#Orx5zh4 zlU(^I9ua#zo)9`cmCW5Kvt)91pz~0b@&G?Uw2oD%2yV27VTW}>Eenh@0=U_{(9%HS z*C(a5G=1JvO&8Gjti7os4ro{Vz)^K%IlS?fIYb%(zC8>f85Ll-9YkHMM6S$>y!cYT z1!SeBmg^~lOVX+>Lz83WdPQ++h8if4oWH1slf@6-32CtPG{~*G_I6H&G&0VYX-=$# zq7{EUG?nMAbXe7^NV!fPq7}KKeYt2&Fi7xVgvFQ%z4Z~Q27(JT@Cadr_?d|J;tJeEN9xPppq8Bu@=l-p?5xgbM{uJIeJS-PkEfhDz|l3rh3e{N z6Cl11KlvT7)QQ+Xl`qK>!Ae6u1K$q+%+?(XC?gGoN4>bRfpG6Fh@Q{H2N^RdDSz> z9#GX){2iX!;5fyiR~cPQ9@+BDz*xjn<1~BopQ?g3p6ZM_OE~H2fF1hvX;z=qfH<`i z_cPC*N)R{+*jZy%z|hj71bRpZ44Wm3Hy?9bl;fDtL3zH{a`}+!);WGv8VBmF(Ag<5 zvs#%3Mf|+(y)9->pV$x9Ce!7TyyjVegn{&u;Sw~l<2as_WBAt>PSk88Hc28D;TW4s zN>HnoZ$=YxHg+OkcX|B&kQ=@aCMH^UV@sD1ZauA(hjO!9ebL?KskYqa;piGWM1P^y z1@Y3$$V5t!4}m9XMbDLXadOE(9L3v26t;yxGY;P}ZbMx+#Gh<*J5>WKi==HW>GtE- z0k&s-L-LJ4?!0cLr4X&4>&$rrPIuZCHv!tRJ0`AyV#S}yU?7L`D3Tn$iMEOF*nn=M zIDL9;bkMPXrQN-JL+W@>%o%^wD{XBlQ>A)+uI)nFTA&;MYtebFrK1q-&0p9k<5VSF z@?(|%Gdp164bk76uKRMb82gs%moxKY-syEm0U^sI38*rKAiLv8C(>6E0j2T zI4B48ksbj&V)aN9gVR@x`Flb*{v`D=w&v8`MavBqkxb>4 zc~+y2AGRQ?Uck}=nxIDfq{ zd;hm3d8#P^Q#M5dNa3yGk(4=vl=k;PViIqw%R~LT4L*_kZ&GXvChe3)^_otV+Nkxp zwzDTrd>n_#DJ5!~)aSi&x9#_%1TxNL3@+q9!#3q%)Z6q{Z&kvpb?l?tz!i;sptI0` z;AF`$Oag5*)Xjp3N;T0yVn{^qBdF6h)Ck_Ue@nNQF+6W9>e_E0mrQRrBSGbVt!`LH zuaedju6j`$BvedYKBHA2ecp)#x8ThyKcL%t9zLH^{mpC>c*G-&;?>pDU6Zr|Y0WCHAfrOseG`WZPzMHfc-H0N> zQRK|s>|TkRlvYl_B)9L{Z4^4UG~h9l=gDh#iMZu-lkUBzpq3oxA;FJohjMo;j41a3 z22P0kqTrNq(`H}pKIwGX*)WfYX5tw$?mhDxE^3s-%sce9W=+wsS7-imPiGXkgDsM6 zowj>a_V}8QTB;`$Cr&tw#D@sFvE*wgI#!HW@wE`#gc6z(W0-fGSMu^44^NHXUmRo} zjD*Umr|s!tcFJP7>E7ch*6h#Me$J)$ULRJ>%&@s^%fD<}tyI4m=q(~k2Yj_PL@fOF z-`+Ipi3#=$i7;V#TQ|nmYadI+(l%B@20A_0h7lYrR>tmoXD6#*RMKK+TbdvI&Ek5E{W>TYiXL>cS-q5P9fP{aqMdq{g1fQ4~^4 zB<@ZMjpvP~FuYacPKg{Q#;1f<_zn4dgEE#2)(9QXIn~_#_hpayOcnnri%k!k&iK@o zdA4n#?9<(2(yYmL*41h6&YyLQs>SNJho)Ae4!c|Z%WeB2;_`&pQAN4O*{8vR4$N0D zhhEvoTE#EP8kJ#M$`|397jd)iTV#!BqUZ3uP!M?TMyhw0K{W|snIa!*7SecH%O+)y zBlwJ?4(CCz>xC!&*J+O?! z=_McM8)pWN&%c)@;2I1TcTq~;%rhf|p}0Xdve(0rcre)J-M@KB$(rDbbK2Cf84qho zMTpD#+f}g3mc3wKOn`4>|5XdTK(4L-4S9lNkMn{)-voy7QmHX9to!YvVlg8UCxLVY zCbRy9nS}dFo>PfqDk2WfN!t592XAU}6~Kvfu+A9M7_x(C79i@#lgQ}p&DhNj64FI0 zI4sc8w=JauYjuSK_t@mZnt)=kVrjm4!>34cswwp-vn0%WlVZmhF31ZR7Ptv|}&DCmE8RN2m3rG}~5+ z07c@dPb{WT!B&%LSTsSexqny^i$20G((4$QdvnGZQjq(XfnQV=5rgQdCUmabx9?zK#wco#!O>KX@_k^Je2Q$W*QEtQY*y# zP3qZ{M%>vS@*3Ru-N0RMn#E>5)5JJTgIn)vmpeMhqMH8acp{Uxy3Kv#BhBFt{omz% zZHuxMCX74Hf`Hwa?!BLx(O6;Zh{oh1 zk9?Tm2WBR8GEiCj!Ywjjg5qkgkPm)OBVoAa0Anb-81s@YwA8POu|YybRh{Z;Y(#=@ zawHH3n>7}m6HFy7o)u+jG#HquHrn`{XwYP9Kbp>0P{)$LPq58;1P&37^OF|AYi;g( zE16q5W@YMaw(_GY8gy8eh?GsirgiJ?)11BHon@2 z2k?CyXF^c}@a~onwJ2e|$bbMr`g-rOR3+#ozPd#1YrHd=nv`(%_VP<2+PIWPF9N9H zq+6r#yodRe~GJSDxd?Ysbs(A`;H~ z2cshGOmhy@h`h}Qg0l#en1aR&tgOq58Og{h_aT_b1|_!y{)7i=8)AC`425Fh09Ef; zN&2hR2k%RQ-Ib&6T}w&$)d#LE`~BN1n`xW2bBb!JP938R*}P4syXwi|1=W+q`;6tI zlglY7sem`;(Egfr5sE7uEVom^we!@iKGxnxZ#qanxh7>x2W2Z37J++aIyhFb6i6i+ z-%r|}!ZM=pgJka17$qBs#RWv}k&v)mVoP!e>9*5Rd|tQtLODMmYupBbTRto0vVNE~ zL@KHU%7Ug+km4GhdVO;$7N^1Z$9eElbk#&HRa2IB$&aL6F+ZZ~-%K8_&lArt8ZFNa zZ>>@-;66ED@^3F8hF{M-hN49}Z?RN8x47e(yE^-6Qr1~~``1k+jokRzdZJ#T ze?CJnKrp8Y165+f+?bw+@_Y?%u-$k&ci>&Vc9##X6b%V5UtVQ*F}#yDp3kS?#jw{a z&8gS$#pxj?^)F+5IVA)w(M>1t0UW|k8er6zQ)6(%j<9)3`6h+jSR~?fvI3fPVJVM+ zwCN#RBLikE)5lbgaD2zd0Gq_Nk%QjTkTEbwie6*tgDY65K~K&^CzhMnZ1OIY#TcIE z17&d65gVw?>P|QcQFP0(gEe1c%<%(p$kg7L)n0cfC3mJtR?d`sGa2(^aQ6>ISNN?a z-J^~O2SXiYVn6bO#&kDj*^5@Dq(FM5XiX4+0uyC;ECk&Q7&k8-5s%231WBA?$q0a9 zXMy6;|QB#W|+(v zO`d8rhA}$HuBy9OscnOYCeZFokYRpi@1bRp-I_&4qY0mz)dv8 z#psFjfRS)w6fSp|gt2NY0OR?&ol6BnpGjYkiYa3CnjR6X!%qwmPg)L#a&-Nb{oV2H zO_$lCeg)Jzczqn6q+{^q-BgdzhMM-Sbi>iS0zdfdq6(c8zG7_{jgca5gy~#3d7O0} z#=MarJ;x^wl?0x2m=3AZqWyJqK?Ge;x4qX#DpG8$R4pVvS1%z2%!}@Idi(P#hs=l0 zbeX2*YrM|Dr`N*!Ifv|L#sj|afrtl@aUa4)SDlXmz+EP`&5FD zH^4h6n@v8B&1dA=lz<+14Z?%#FV_l(PX(uP^O83`(#wDb`dpW)0(y8nGWxbRTN4qg zbPU*fXZ^u~Yy|M%@qq=pIZX~a)a<1{R}ixEQ{PwCmvJcSi??WZ5K>LnI@Cj9K={AN zbtd=RRU~KDiP{d~1tc=>BfLc^!n7cB9`KcuG*3h%hC>>Gc-FqGJ#D{Az`w4n z>;DvS&)uSF;os}x#=WTf%HmFzK>{QbkiW!_RO6LL>ck8dr}b%)tf7M}m$@%eVNR~$pjWIY>)K76S&6D)ErTYo$!HbpW?J(LEb1Oh$ZHwXN1VXL70mn0hQUgw2^-o1YBD=iZc88NCXQc; zG}na7)C7!ox@$qVt+U6?6dipyH+rh4^T|;1{c5 z+KB?(kr}w(*g+=mOvH}!!q=G z_xI0Tg_ykAxA`S9xAJZ$P^cB4EX&1`Ps=_2hRR4R!B zePQ~o{hbjJpb3KMMZsq1*J@(r{ltu{JFT3YkH>GUB1~8#?T>dK(ZY)hUEV?TAckZEm<8m!rW?ciPRR}Sl6Yh7Qq z@;hYn@cSF`r9^T-)LuFshVKpK(d^`c`5B{_nCxn(lLIv0F)EirmwNF7Guoeyd}Vkm zve@n34B@6edk^VE|A2|r`k( zRg-Mi;u||Z`OySCTK3@T>(UrSTgPBLBFc4pTFx2xHmpm;PO3L5{mkDGSOUGEZ$3!5 zLj6t*e#X8riT-kd@x-b6y~G?N@rX2u5QNA4ld=4cAiA!g#TjIOw^LMNR>9B~k5|tu z6}X36Ay|b*C|MGbBT5Krbc;*8Q(0;IU@;5{`tp^#?0HS14m5^2BAtv7Jr<^r1yQGu zP|-$dQdV_YmC&%Ml2j@pjzKzfk)XN2JhaOcS<=ftV9^@Nn9S(0f6rT0GqeX_^pl{X zRfjUNPfT@zW|`PwNr9da2U{AeQ|S;=R!Bq|Ku^+a?TuGF-A+MX+36CbQ(Z{d2zybS zgye5ZsWq(9HY{3t;~hhCbOvo9fcxL?@`w;9S0%{PnBWwuFQv>o!S4U=j2?e6q-vl@?G zk~X>MqMKZrw9{AkYtz>yuM4k*q2jbBOI6D#~xqViag*hj9#4yU#j=25+6~h{c5z2|Mh?PZe?Tuj&(Su5)z2AX0V3TOflX7$@yQZv$<@WkFiv(@D z#q*Q@2#_7oiKZ-KGIjCmroEgtO4+{>u$!qm+{V4gJ{&}%Je;oN$4BHJ??a?9w%Qn+ zA49Rv&qUp;b?CTvTi+K}?3$;dHhk{7-etD%(>%^w>PoIidH*fMSkYjz`n>h_E22eH zWP2%hnp{~e%kyA5zbbm8eiQY;R^eibVl@I|K36Ttm7u7d>!RA5qLM;xI$|Rk0aF2) zkQ08N{@vimdl`nE5-VHIvD{d2{e&fI;$>lRo}pCOSZNvkO>;G~q>pM-A9rCpgMP$G zWLM)e+H<~}Byt%;WYf|m{|=_vht2D&3hH^7!^#E@E6t+KD;tAYn#PR=w}VOBPmEg| zFVg;q-Ik&r)BN*&9N~=b`kPs^IpEPMVa>&Od2zB@(r!B?A2Ej(DT!k^ul2^#y-_7Z z7?2%^K~~D#ZBVWkJ>OxDi3|>V;#!jCPOm0`OW1~)ECr_^6%~w4oZvjvP)Dl~9p%1gogfOFu6PbC5kIiBpYj;{s!w655Podi3k^ zSY;L!&rb1E6)u%b+IgZ(lfz>!iiJVA5lsc&LPq;}hTQHBWee3>ZNv3Z=n~29XfgUZ z7@9a>q^mm1nTO6E=P`_GuWN{RTvOTsRy`GBffl_SeMb5?X1EsJm&1tL2X=EcYX5|B zgnsne&jRtH8Z?rnneHz$2@{_;BUU;!Ix%egsGc1LxW=C?kK!IH2K&VTG%km2N={MP zDu@Y3Rmk8EE|=^HZ+8aS`10U)bO|FJYMbA?RzVEQBlp5+_bOZFBdnZKqtyEfg7Lyl z4adqX_*%-0bpw<^A!!js3?@B)M@#atJDMOHk`m9qL}&iI^s8^z37kB^6nF#kbL}L$ zhp+R=>NZ&qczRWV#K5@2uE2C-@U7c1kfcUQ(5*<%NA9NzM&W78uQf2@albRKYyS&t*#b-9 zCxDExUpqG^6>dJ+N<1@{U39t94_ILuf_0O~AYIG;^>%!k4{xn!`(kA2|5O_x$J9}n zEmE7PW<)Uw%m4_GH>Y)d(sb2|WrJb|iOJ#9+XSU+53T9)rL0@K-*{#g>M~E$tPw(A>A*=(>X}~13FV?jQPpzRnmN~C|6*YBW zklLeHW@NO5Z)YrGuPwGO*R`)bsj5{y0u{S_4cE3JT6iVS`Sj<%N^~Zz?qHb8VzPFM zTOov74bZ1&W@=h`Fzm?fb}Csc!CweLKugfg|EA$!Gp|#fNaj8i*c{;o+uGdA&cPsH zlIW9@|A91NkcXwDplXVQX!DQ)ila%e8v5}3H)1?N3CNYLwbag@wLZ|9`)VK6V{j8Q zOd-Hf*EiA7f+HJGAVLeFm?rHg`Yc~1X>EkG9^Dv>XypCXxJYw0NMF?z;Ru_?V`rr9 zuD*C)vplMXD|@OUTP(PJES$X9Zu-u%ncLiKl35Mh7OvM6+ZV>pF5Z-j^5&oz|MGOX z=GQ#pe|gY1+g?x9)b1o8Ve@=?e{p-crf3tlx<0R?{@!#!x5dn!(bpKO*TuG#9(Adb z>mMSqiR!|`@m#6dYI2BL(0(UDHJ#<~#&J1yp~+OAD2ozOJxY`SG^+iZj04%zZ`J!W zHHkAIL;r+~$hJLV(0FbNIb}6HTpN+p)`3P2D+kuBpz$q?ozCf-V-sa{4u8VqWQ%m8 zRp7qc-EU)R%2NQl-9VK_Xl`g~qbSPDGvyx>IKg%hk!W|WysrV(81RSC$C@~NEhoAo z6#-eZi{*D9_f{)6I18^4|F8fp%16TI&tDp?FL&%rBYne-$ly1znJDh@%@~A*!?pk^ z$|;f?=ylF6FwFvS-=0y;n+I(2l+!Mxk8~J8OUemtH6*ps?Hp)#bUPns@EdOSAdcnvO?&cBxRLd z-c8puf_=_Tv!OSJ4~py(@oo&m0@>14&?UwKtrqYuz$&~t(n~zbfzg+$NuhNY9P)Bz zr)rGPm8i>=b#Fb_lKE?m*Y2L@lLZT{;;J_t@+UYN(c3jTUVFHE5W6{Scd{>ZYDAi* zt$FzH6gjxF4a*w@#CsuwwB12*hS80^S^`@%ZzpV;1o1ad_Z^1enve=#4b@=3E znJ=I+l%sH}YHV%F7)xSoCN7m^9iCC9eOjk-_nx{9)kb4cFt@wt*J=SL``S%4ACo@n za1@J9nI&*4oH8=SA_pGTclike?rlZDXP+PW;pqTs!aY2pgh%cl1IntO`9w}q&VnQcj9M@Rsh3=x6Mu?_G{(GY zby#Ytdq!xOqkSHU2#-)$$&dnIFr#tJCo9c|1RSm;4BWCwQ%Jm8qKHv%swi%1=gu42 z4ELwEFBh?KMk|r20=Qf8*D`JY7!R2ue!tCGUl5%)`x@lA@+UmkXODnW-V+N7$mT_4 z);HKUib%U=K2W77KDq?~q!bvC{;%FXungD)p|19n*txf1w9Sv9eG5s+oPXGwyv~a& zs#faFU&SgRy>F=J1m5S`_dTNj9I4t~>o|fgoRl>1|J_9|Wh_^1Z=7N5@$51j3?PiB z#f^L-Zs}MbTD@e!Y(S}rA{jAgrXa}*j0Da%$W##b9^8;KU~OBIOH^?-e6^WeNihdT ziPXHKHoG8~Z41%*(v4TfPe&n()yErElCgCfxz7kfRFt~~slt}UCyq%BS}GI?Xzz{} z4MRcUC5-LX*GhQwV>!%c{ldLUO;Qql{iqih)zZ{waPl(n+ml_sD@5wsG)8JFc*qe< z2Gy+~+JJT`VJLH?u--2+IE#*Wdy;>EY%ZkHp78V_fSxYB{#?9Qi8FJkZmW0i#TxMC zIB9xg{{(Yt)+^O|UhHl71Cy+>sPC8t$2pmYc;f+`#toUuiayt^J!hihFMz{jg0Q^M zvga}|vw#J>1hc)>MZ=BNAhNQ5zNXyRU>i`})luG<6Qxfw|5Om1ogK-1F9N>g#e2&G zu#`RXE>=j(s-U0D8}o$0{{CzX^j7c<@H&|vhUVPS$+1hO2zs{)0-3TOoRMdaCC`=F zAKR48D0?_r2reI}-2t=L6SP&!Hy8BD5=vur=)YLSHhvnm0Gfz;Wzg<-xm ze1%lC6#&fi{q`N89g}Ofx&z~#eOV8}u zf`^kf*Uv!`6t_yWNwh}K@9RcsJ}ENiRs6n;%H8K|G}N=2(kwHYi%k^Ws50a=R#h8~ zgxeJ@+?k4-PVkdP&bXyN7$(Xg$%RzqAk95;xoe0006BO)ynGqiyuYe~Co;tR62#YB z>U5WL`P<-{z;sDowb*n(;JBOFgyP_hi%r)% zIJ1qbh9DzClTf15Zvo)=>opRhCN80LG}fI6x;d&R*@=_v)y7zK04TP216M(Bpf1+QvxAP2<3 zmzy)@XiCJWn8_dtKEs{-%P&}7Moi%D3ZV~3D>y#|u`58zKe*1TG2umydw*BW(Sw?X z%go}e=M?9Fw&%eN!dL&;iMTFP_U(|N1|d5Fsmm!XqkS7b@V02=`*uz@C9fgHFky^0 z6eG;jm1aOZ#3LSL$#C**5_oqQK3@}2_#9{TvzqYs9Pv@)w7}MFTK!n_vB0(YQt$|< z^ymy2L6zGUc|E=3l%oCyF*SgCE7Qf&y#OZj=U;e!0s>iV5SP24b4wA)6slbkKPqVa z?L7vIXHveS>h38t5DB(K7mO+b>$HL{jmcsulpV9gIQ+x8|K(jy>TN9DWHsRd-ESVJQ5c}`_fCcA#g-Gmp zL9`a{aW52!x-Xv(liSJ&(t9irNI!(V-XjjUhIaKPVf1eo_X~Srh+bxvmvd1SB{2vp z%wybkv@OTW;}j214>YImKO4Mx*VExQxs$uc1oj(hCj=~pPXQce4-mYN3K~rT&4clb zV5Q3QA)*t>xFc<)$Gw1SYsK|7B|$F-FRzC1FnhN_gFTQu|AQqEncRzh0Z6B{M)+C< z?u7TwN`dnG0r#=owToakaXE%{HxfBuQy5p=EZ(YlaaVUr2=-6PP)+q>>hzs585^st zY6X>ID{0?7@ z=h44eJX;z{S1wJhYB!nt&1~C_TX)&^X*2?!zN!SN1c%|6_m5ayicG1(l*Fy;#;DzL zNcKsqTvA%YiB)@?rim}#*ZBHl+u8^>-_NuAuhV<%)0+B}?EN!mTw3Dx*D$=fr${(d ztqrI?OuuBAvJdwwJ4{1s#VOB+F3a$^pK;jc!^>uQA}tp0M?tagM(|)71f;VY>(F>& z5E?p1FmY%imeRp8ba6QUHQK$*NNA)javS{-@X&e zvtv0<#1x?N>6t|SePNQkwwJyq(K<7g@jJmdML2nT?gZO?nqU;AwC0{U8(w-dM`0*L z>xv;G(}c96S4)A_{IyijaH#&KvIJB`3D48TL;Ez}==}t%=T7tmytIby6cLutzXBlT zg%rq64!uz)`MUkLozQE9WyU#Ua)^a8;n>HbA^Aw^JVulCABWe7wT?Bmsmbw%BZu9l zbPU79H^?Pg&By<#ThlePHJnSOr_bI#q72{~2g`-%U$yB@=|A~a`97}QGD-s2vty+4 z?F!Pw8XCm3MuY0uqe?= zSwbc1gbRN{l5YYTfwFkLBUr^3bqOrHY;3XDO8DMMEd;wD9o z0A%eejz)}V2c{GY%pwWsd*cO1^>_UGe)vX~t47NI;2jX64Mv7}g@FM$!j#4Sul`SW z#=nm)7`WpG(9a%B8>tW}6R9039@&6FOZTN8uXkrKX23C2IrI@q5>*s#1UC+%g1N-D z1h%AO31q2m$!!U~l3m+Sw_b~0H?7ax{}s{iTM%x5NCr}ZRf25-dkjwlUCmZ4u4&Q2 zV|#9=YD>HC-9t2}IOGtf8q*v#9cqKe3*L?AgY^yb1@hqodI7oy3J1}Fc!1o9@PHhN zc!8)%*dlwAgpd>K7aJiLDHk$>mFLl?*(cto7^e?279nmX79uv4q)u=zd4NouMx1OEGTx(5t}jn}~>T|FSoYs}qzy6e$!tlqAX&xu>F%JdA>+;zr4f z^e7*Nj9Ks;rV*SG_#xFH#h6FpcIilIY8i2Xp!d`Cg#4)@x5w9&t&5KU(>mL;#=D)k_n!<{DfwCzCKT@`SI(eT5`YzvG~WPcZM|H&2*@KD4d z>ZZ&d%IB$Z4elssli^YR@DKb_?x&>sq=6BfclO8%R(xFRQh)rr5*PyK-r^5}4GT(l z(-Y?(M64o)+Qlq4z`myGQhFU9)CHLk2ixKqNeHfUWv*$V*`7&Ty0JGoEhhl9&h-d* zXUnhVqeXXu3;AMkfGcaZn+#+$P#2ewEuZhXC^A9#t1B5K2yqA)1ge(y_I3?h7njx@LRV0N zd5f!)3@xoilPpGM9cc?qi--H^K9$+G?rEJWw0(?itnKuT^gd8DgWm~inIvlQMQZ7z zQhJ!lM(oKppOa9PBNCMpe=5h!E2pq3NB>q%a#W7HS5AXjj)+)JkXnuzTTY=_j;dHr zvNS^e!j<@Aj@93+Gklxb6P7tJn%U=QOqZa@9;Kc+WqCxG!k9XomN^Jv;sAHd zkaN$L1KkoEq1H2~*;k}Fbg0>zq&c{#+25o&{J7B*wJ|Wc(O0!Gbh*)+wK2H4(cif- z{K?f5z%|g%)mOkZw9nO>z%@9})!)E1eBaR%(J?UI(O1zibWU{uyLCXlb%eWh$h~z8 z!gD~xbA-%u$jEaH-E~0Ob%fn@$k}xa?tMV!eT43P$m)Fz|CPz+we-=-$dIZ(H*%47 z`LytqPrY_o7p2jH+w4f$?2O%f{($h%u25c}K0$c|{f`>d{I8W5{Qp{` z;u^(eVpm0@qI=ha=jrR%ebO=Iv}$&Zr>s%Q9d}aan6^>PKh^cJ%LQk1&Zew28LN_i z^DAbass=T6%PSTa%uiSzQJq8D%l{8;TKoUrY-S?53a(E$-=e$b@!mgozD_vWqN@we z|Bo}QWPIVw{~yaPI6h%_kN*F<`CG030)I4)=;(s&#O!&yvAS)K8t;Pb6V|t=|GR7A z#uXi&wR6Pzf8#Lk*Bj=s9lzdfc + - @@ -66,10 +66,9 @@
- - + diff --git a/rd_ui/bower.json b/rd_ui/bower.json index b7ff4ed63..6779f431d 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -16,7 +16,8 @@ "angular-resource": "1.0.7", "angular-growl": "~0.3.1", "pivottable": "git@github.com:arikfr/pivottable.git#master", - "cornelius": "git@github.com:restorando/cornelius.git" + "cornelius": "git@github.com:restorando/cornelius.git", + "gridster": "~0.2.0" }, "devDependencies": { "angular-mocks": "~1.0.7", From c24b6cabc9867f72b62246fdc55b7df1d7b8dbab Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 19:48:20 +0200 Subject: [PATCH 006/540] Move cookie secret to settings. #7 --- rd_service/server.py | 2 +- rd_service/settings_example.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rd_service/server.py b/rd_service/server.py index bf2cd5328..39f84a880 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -298,7 +298,7 @@ def get_application(static_path, is_debug, redis_connection, data_manager): static_path=static_path, debug=is_debug, login_url="/login", - cookie_secret="1f7146d871a6a23263089419cc443813", + cookie_secret=settings.COOKIE_SECRET, redis_connection=redis_connection, data_manager=data_manager) diff --git a/rd_service/settings_example.py b/rd_service/settings_example.py index 149366dbe..da2e35409 100644 --- a/rd_service/settings_example.py +++ b/rd_service/settings_example.py @@ -20,6 +20,7 @@ ADMINS = [] STATIC_ASSETS_PATH = "../rd_ui/dist/" WORKERS_COUNT = 2 MAX_CONNECTIONS = 3 +COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" # Configuration of the operational database for the Django models django.conf.settings.configure(DATABASES = { 'default': { From 7280b6c558fadd29343511f5b8d79116f3cb0034 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 21:52:21 +0200 Subject: [PATCH 007/540] Fix: sorting on number/dates columns was wrong --- rd_ui/app/scripts/query_fiddle/renderers.js | 35 ++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 1594d1ba2..21fdc7c49 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -104,7 +104,7 @@ renderers.directive('gridRenderer', function () { }, templateUrl: "/views/grid_renderer.html", replace: false, - controller: ['$scope', '$filter', function ($scope, $filter) { + controller: ['$scope', function ($scope) { $scope.gridColumns = []; $scope.gridData = []; $scope.gridConfig = { @@ -123,24 +123,16 @@ renderers.directive('gridRenderer', function () { $scope.gridData = []; $scope.filters = []; } else { - $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { - return { - 'label': $scope.queryResult.getColumnFriendlyNames()[i], - 'map': col - } - }); $scope.filters = $scope.queryResult.getFilters(); var gridData = _.map($scope.queryResult.getData(), function (row) { var newRow = {}; _.each(row, function (val, key) { - // TODO: hack to detect date fields + // TODO: hack to detect date fields, needed only for backward compatability if (val > 1000 * 1000 * 1000 * 100) { - newRow[$scope.queryResult.getColumnCleanName(key)] = moment(val).format("DD/MM/YY HH:mm"); - } else if (angular.isNumber(val)) { - newRow[$scope.queryResult.getColumnCleanName(key)] = $filter('number')(val, 2); + newRow[$scope.queryResult.getColumnCleanName(key)] = moment(val); } else { newRow[$scope.queryResult.getColumnCleanName(key)] = val; } @@ -149,6 +141,27 @@ renderers.directive('gridRenderer', function () { return newRow; }); + $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { + var columnDefinition = { + 'label': $scope.queryResult.getColumnFriendlyNames()[i], + 'map': col + }; + + if (gridData.length > 0) { + var exampleData = gridData[0][col]; + if (angular.isNumber(exampleData)) { + columnDefinition['formatFunction'] = 'number'; + columnDefinition['formatParameter'] = 3; + } else if (moment.isMoment(exampleData)) { + columnDefinition['formatFunction'] = function(value) { + return value.format("DD/MM/YY HH:mm"); + } + } + } + + return columnDefinition; + }); + $scope.gridData = _.clone(gridData); $scope.$watch('filters', function (filters) { From 81c59869e0abd5dc30e5756e2fc184534ecdf8f5 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 28 Oct 2013 22:01:18 +0200 Subject: [PATCH 008/540] Fix numbers format in grid. --- rd_ui/app/scripts/query_fiddle/renderers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 21fdc7c49..2878850a8 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -151,7 +151,7 @@ renderers.directive('gridRenderer', function () { var exampleData = gridData[0][col]; if (angular.isNumber(exampleData)) { columnDefinition['formatFunction'] = 'number'; - columnDefinition['formatParameter'] = 3; + columnDefinition['formatParameter'] = 2; } else if (moment.isMoment(exampleData)) { columnDefinition['formatFunction'] = function(value) { return value.format("DD/MM/YY HH:mm"); From 43d5b68ec8145d4286b99405dedc5bd9220d401c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Oct 2013 09:13:07 +0200 Subject: [PATCH 009/540] Enable cohorts to all users --- rd_ui/app/scripts/controllers.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index baab47318..1fd1523c3 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -33,11 +33,7 @@ $scope.$parent.pageTitle = "Query Fiddle"; $scope.tabs = [{'key': 'table', 'name': 'Table'}, {'key': 'chart', 'name': 'Chart'}, - {'key': 'pivot', 'name': 'Pivot Table'}]; - - if (currentUser.is_admin) { - $scope.tabs.push({'key': 'cohort', 'name': 'Cohort'}); - } + {'key': 'pivot', 'name': 'Pivot Table'}, {'key': 'cohort', 'name': 'Cohort'}]; $scope.lockButton = function (lock) { $scope.queryExecuting = lock; From d4176e653d4c3d1440ac374a5c25e476e0cf26a3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Oct 2013 19:08:11 +0200 Subject: [PATCH 010/540] Group dashboards by category. --- rd_ui/app/index.html | 11 +++++++---- rd_ui/app/scripts/controllers.js | 19 +++++++++++++++++-- rd_ui/app/styles/redash.css | 20 ++++++++++++++++++++ rd_ui/app/views/index.html | 5 +++-- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index dd0936185..e6e3c77c9 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -37,10 +37,13 @@
    • diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 1fd1523c3..c6b89544e 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -184,7 +184,22 @@ } var MainCtrl = function ($scope, Dashboard, notifications) { - $scope.dashboards = Dashboard.query(); + $scope.dashboards = []; + $scope.reloadDashboards = function() { + Dashboard.query(function (dashboards) { + $scope.dashboards = _.sortBy(dashboards, "name"); + $scope.groupedDashboards = _.groupBy($scope.dashboards, function(d) { + parts = d.name.split(":"); + if (parts.length == 1) { + return "Other"; + } + return parts[0]; + }); + }); + } + + $scope.reloadDashboards(); + $scope.currentUser = currentUser; $scope.newDashboard = { 'name': null, @@ -203,7 +218,7 @@ $scope.archiveDashboard = function(dashboard) { if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) { dashboard.$delete(function() { - $scope.$parent.dashboards = Dashboard.query(); + $scope.$parent.reloadDashboards(); }); } } diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 391370a52..51257114c 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -106,4 +106,24 @@ li.widget:hover { .CodeMirror-scroll { overflow-y: hidden; overflow-x: auto; +} + +/* Because of ng-repeat we add span between the .dropdown-menu element and the li element, so we had +to add those CSS styles here. */ + +.dropdown-menu > span > li > a { + display: block; + padding: 3px 20px; + clear: both; + font-weight: normal; + line-height: 1.428571429; + color: #333333; + white-space: nowrap; +} + +.dropdown-menu > span > li > a:hover, +.dropdown-menu > span > li > a:focus { + color: #ffffff; + text-decoration: none; + background-color: #428bca; } \ No newline at end of file diff --git a/rd_ui/app/views/index.html b/rd_ui/app/views/index.html index fc4f3d4e5..2f7342043 100644 --- a/rd_ui/app/views/index.html +++ b/rd_ui/app/views/index.html @@ -1,7 +1,8 @@
    -
    +

    Dashboards

    +
    - Dashboards + {{name}}
    From f1da4ad5421d79cf19e25eb1b7c02f124612a936 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 30 Oct 2013 10:08:22 +0200 Subject: [PATCH 011/540] Update README.md fixed broken Tornado link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7a9f4608..0e928df28 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This is the first release, which is more than usable but still has its rough edg ## Technology * [AngularJS](http://angularjs.org/) -* [Tornado](tornadoweb.org) +* [Tornado](http://tornadoweb.org) * [PostgreSQL](http://www.postgresql.org/) / [AWS Redshift](http://aws.amazon.com/redshift/) * [Redis](http://redis.io) From 0c5b0c892fb9efc1d756addd50bdb062797cfce2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 12:23:16 +0200 Subject: [PATCH 012/540] Add link to demo instance. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 0e928df28..e96a91d23 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ Prior to **_re:dash_**, we tried to use tranditional BI suites and discovered a This is the first release, which is more than usable but still has its rough edges and way to go to fulfill its full potential. The Query Editor part is quite solid, but the visualizations need more work to enrich them and to make them more user friendly. +## Demo + +You can try out the demo instance: http://rd-demo.herokuapp.com/ (login with any Google account). +Due to Heroku dev plan limits, it has a small database of flights (see schema [here](http://rd-demo.herokuapp.com/dashboard/schema)). Also due to another Heroku limitation, it is running with the regular user, hence you can DELETE or INSERT data/tables. Please be nice and don't do this. + ## Technology * [AngularJS](http://angularjs.org/) From 8c9d8310ca92d3d5c30c300d1aa082fd168adf6c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 12:23:49 +0200 Subject: [PATCH 013/540] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e96a91d23..5564fd341 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This is the first release, which is more than usable but still has its rough edg ## Demo You can try out the demo instance: http://rd-demo.herokuapp.com/ (login with any Google account). + Due to Heroku dev plan limits, it has a small database of flights (see schema [here](http://rd-demo.herokuapp.com/dashboard/schema)). Also due to another Heroku limitation, it is running with the regular user, hence you can DELETE or INSERT data/tables. Please be nice and don't do this. ## Technology From c60c8438e11345967a3ad68e1d1eb15e3df424ef Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 12:33:29 +0200 Subject: [PATCH 014/540] Add dist folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9de958bb8..a6086e0d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea *.pyc rd_service/settings.py +rd_ui/dist From eed3b4a7cf5e8eda908aad59a8f908c5bcdf8b3c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 12:37:33 +0200 Subject: [PATCH 015/540] Add screenshots to README. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5564fd341..2fee1ebc3 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ This is the first release, which is more than usable but still has its rough edg ## Demo +![Screenshots](https://raw.github.com/EverythingMe/redash/screenshots/screenshots.gif) + You can try out the demo instance: http://rd-demo.herokuapp.com/ (login with any Google account). Due to Heroku dev plan limits, it has a small database of flights (see schema [here](http://rd-demo.herokuapp.com/dashboard/schema)). Also due to another Heroku limitation, it is running with the regular user, hence you can DELETE or INSERT data/tables. Please be nice and don't do this. From 7bdd3074f5f96e9cdc4e8d53f203e25299d7c369 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 13:10:16 +0200 Subject: [PATCH 016/540] Add getting started instructions. --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2fee1ebc3..8eee5f588 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,41 @@ HighCharts is really great, but it's not free for commercial use. Please refer t It's very likely that in the future we will switch to [D3.js](http://d3js.org/) instead. -## How to install +## Getting Started -TBD. +1. Clone the repo: +```bash +git clone git@github.com:EverythingMe/redash.git +``` +2. Create settings file from the example one (& update relevant settings): +```bash +cp rd_service/settings_example.py rd_service/settings.py +``` +> It's highly recommended that the user you use to connect to the data database (the one you query) is read-only. + +3. Install `npm` packages (mainly: Bower & Grunt): +```bash +cd rd_ui +npm install +``` +4. Install `bower` packages: +```bash +bower install +``` +5. Build the UI: +```bash +grunt build +``` +6. Start the API server: +```bash +cd ../rd_service +python server.py +``` +7. Start the workers: +```bash +python cli.py worker +``` +8. Open `http://localhost:8888/` and query away. ## Roadmap From 03fbf41a350734746ef6ffc36cca30e7edb1201b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 16:38:09 +0200 Subject: [PATCH 017/540] Fix: stop sending latest_query_data back to the server. Fixes #31 --- rd_ui/app/scripts/controllers.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index c6b89544e..5fa5136d8 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -52,6 +52,7 @@ if (!oldId) { oldId = $scope.query.id; } + delete $scope.query.latest_query_data; $scope.query.$save(function (q) { if (duplicate) { growl.addInfoMessage("Query duplicated.", {ttl: 2000}); From 2017a2c9db50fff94039d2b6d5879e3ca4d0ddea Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 16:43:18 +0200 Subject: [PATCH 018/540] Fix #18: don't retrieve query results when already have them & ttl = -1. --- rd_ui/app/scripts/services.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 946395db2..d66277f09 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -248,9 +248,9 @@ } var queryResult = null; - if (this.latest_query_data && ttl > 0) { + if (this.latest_query_data && ttl != 0) { queryResult = new QueryResult({'query_result': this.latest_query_data}); - } else if (this.latest_query_data_id && ttl > 0) { + } else if (this.latest_query_data_id && ttl != 0) { queryResult = QueryResult.getById(this.latest_query_data_id); } else { queryResult = QueryResult.get(this.query, ttl); From cf1d20c958ef93052e0b6f134b6345fff8645f0e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 16:51:08 +0200 Subject: [PATCH 019/540] Don't send query result when saving a query --- rd_service/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_service/server.py b/rd_service/server.py index 39f84a880..738b3e1f7 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -205,7 +205,7 @@ class QueriesHandler(BaseHandler): query = data.models.Query(**query_def) query.save() - self.write_json(query.to_dict()) + self.write_json(query.to_dict(with_result=False)) def get(self, id=None): if id: From c0c2fed6acb56472b934cd297330859541bbb080 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 17:11:00 +0200 Subject: [PATCH 020/540] Fix: when query had unicode characters it was failling to execute --- rd_service/data/utils.py | 2 +- rd_service/data/worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_service/data/utils.py b/rd_service/data/utils.py index 40d938efe..0d28a26c7 100644 --- a/rd_service/data/utils.py +++ b/rd_service/data/utils.py @@ -20,7 +20,7 @@ def gen_query_hash(sql): """ sql = COMMENTS_REGEX.sub("", sql) sql = "".join(sql.split()).lower() - return hashlib.md5(sql).hexdigest() + return hashlib.md5(sql.encode('utf-8')).hexdigest() class JSONEncoder(json.JSONEncoder): diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index b0915c215..2579f5ba8 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -24,7 +24,7 @@ class Job(object): wait_time=None, query_time=None, updated_at=None, status=None, error=None, query_result_id=None): self.data_manager = data_manager - self.query = query + self.query = query.decode('utf-8') self.priority = priority self.query_hash = gen_query_hash(self.query) self.query_result_id = query_result_id From 2c21ae28d96f438e25acce7996d916ffe9233383 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 17:38:05 +0200 Subject: [PATCH 021/540] Fix: move utf deocding to proper place --- rd_service/data/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 2579f5ba8..97ad419e6 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -24,7 +24,7 @@ class Job(object): wait_time=None, query_time=None, updated_at=None, status=None, error=None, query_result_id=None): self.data_manager = data_manager - self.query = query.decode('utf-8') + self.query = query self.priority = priority self.query_hash = gen_query_hash(self.query) self.query_result_id = query_result_id @@ -111,7 +111,7 @@ class Job(object): job_dict = cls._load(data_manager, job_id) job = None if job_dict: - job = Job(data_manager, job_id=job_dict['id'], query=job_dict['query'], + job = Job(data_manager, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'), priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']), status=int(job_dict['status']), wait_time=float(job_dict['wait_time']), query_time=float(job_dict['query_time']), error=job_dict['error'], From 783328eebb4d1a744fb6cd42be2acbd4eda9c78b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 17:45:26 +0200 Subject: [PATCH 022/540] Fix: columns with multiple spaces were not showing correctly --- rd_ui/app/scripts/services.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index d66277f09..18f4c926c 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -141,13 +141,13 @@ var parts = column.split('::'); var name = parts[1]; if (parts[0] != '') { - name = parts[0].replace(' ', '_').replace(/\?/g,''); + name = parts[0].replace(/ /g, '_').replace(/\?/g,''); } return name; } QueryResult.prototype.getColumnFriendlyName = function (column) { - return this.getColumnCleanName(column).replace('_', ' ').replace(/(?:^|\s)\S/g, function (a) { + return this.getColumnCleanName(column).replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { return a.toUpperCase(); }); } From 7a1893cc8a728a03ad4b8d661e92c8342b728740 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 30 Oct 2013 18:25:17 +0200 Subject: [PATCH 023/540] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eee5f588..8334d00b5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# [_re:dash_](https://github.com/everythingme/re:dash) +# [_re:dash_](https://github.com/everythingme/redash) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. From 18e56fab57fdfb174173a9d06a894216106fe872 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 31 Oct 2013 12:22:45 +0200 Subject: [PATCH 024/540] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8334d00b5..9d6fb1c64 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,10 @@ bower install ```bash grunt build ``` +6. Install PIP packages: +```bash +pip install -r requirements.txt +``` 6. Start the API server: ```bash cd ../rd_service From e2e61b7c59b2212c319069e1e5ce8584d65a7155 Mon Sep 17 00:00:00 2001 From: Shay Elkin Date: Thu, 31 Oct 2013 17:14:59 +0200 Subject: [PATCH 025/540] Fixed installation dependencies, instructions * rd_ui/package.json: * Add missing bower, grunt-cli dependencies * Take grunt from HEAD, as 0.4.1 has broken dependency: https://github.com/gruntjs/grunt/commit/08a3af53ffe9a0fcb15102088f7407120bd5a397 * README.md: * Update instructions --- README.md | 6 +++--- rd_ui/package.json | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9d6fb1c64..921d062ef 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ git clone git@github.com:EverythingMe/redash.git ```bash cp rd_service/settings_example.py rd_service/settings.py ``` -> It's highly recommended that the user you use to connect to the data database (the one you query) is read-only. - +It's highly recommended that the user you use to connect to the data database (the one you query) is read-only. +3. Create the operational databases from rd_service/data/tables.sql 3. Install `npm` packages (mainly: Bower & Grunt): ```bash cd rd_ui @@ -65,7 +65,7 @@ grunt build ``` 6. Install PIP packages: ```bash -pip install -r requirements.txt +pip install -r ../rd_service/requirements.txt ``` 6. Start the API server: ```bash diff --git a/rd_ui/package.json b/rd_ui/package.json index 325a5acbd..3b476a555 100644 --- a/rd_ui/package.json +++ b/rd_ui/package.json @@ -5,7 +5,7 @@ "grunt-karma": "~0.6.1" }, "devDependencies": { - "grunt": "~0.4.1", + "grunt": "git+https://github.com/gruntjs/grunt.git", "grunt-contrib-copy": "~0.4.1", "grunt-contrib-concat": "~0.3.0", "grunt-contrib-coffee": "~0.7.0", @@ -28,7 +28,9 @@ "connect-livereload": "~0.2.0", "grunt-google-cdn": "~0.2.0", "grunt-ngmin": "~0.0.2", - "time-grunt": "~0.1.0" + "time-grunt": "~0.1.0", + "bower": "~1.2.7", + "grunt-cli": "~0.1.9" }, "engines": { "node": ">=0.8.0" From d6d3dfcb356aa5306156db92b19c9cc1cbf9ee34 Mon Sep 17 00:00:00 2001 From: Shay Elkin Date: Sat, 2 Nov 2013 15:15:31 +0200 Subject: [PATCH 026/540] Take specific grunt commit (that fixes gh-886) --- rd_ui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/package.json b/rd_ui/package.json index 3b476a555..f703c69eb 100644 --- a/rd_ui/package.json +++ b/rd_ui/package.json @@ -5,7 +5,7 @@ "grunt-karma": "~0.6.1" }, "devDependencies": { - "grunt": "git+https://github.com/gruntjs/grunt.git", + "grunt": "git+https://github.com/gruntjs/grunt.git#08a3af5", "grunt-contrib-copy": "~0.4.1", "grunt-contrib-concat": "~0.3.0", "grunt-contrib-coffee": "~0.7.0", From 7ca1b5e761d64743a895ce2c652221c36bc1a755 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 4 Nov 2013 15:51:58 +0200 Subject: [PATCH 027/540] Make qr take redis connection params from the general redis connection --- rd_service/data/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 9babcba90..3af229f30 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -30,7 +30,7 @@ class Manager(object): self.workers = [] self.db_connection_pool = psycopg2.pool.ThreadedConnectionPool(1, db_max_connections, db_connection_string) - self.queue = qr.PriorityQueue("jobs") + self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 # TODO: Use our Django Models From 2b8a3f66e9b334575024181794dee98e61d6d8dd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 4 Nov 2013 17:11:45 +0200 Subject: [PATCH 028/540] Update title to use | instead of : --- rd_ui/app/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index e6e3c77c9..ce59987c5 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -4,7 +4,7 @@ - + From 770a8a8d5714c4aa7e5f9b7439e0beb362345328 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 5 Nov 2013 13:27:28 +0200 Subject: [PATCH 029/540] Fix: when series name was 0 (the number) it would use the y name instead --- rd_ui/app/scripts/services.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 18f4c926c..61be9b940 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -86,10 +86,11 @@ _.each(this.getData(), function (row) { var point = {}; - var seriesName = ""; + var seriesName = undefined; var yName = ""; var xName = ""; + _.map(row, function (value, definition) { var type = definition.split("::")[1]; var name = definition.split("::")[0]; @@ -105,11 +106,11 @@ } if (type == 'series') { - seriesName = value; + seriesName = value.toString(); } }); - if (seriesName == "") { + if (seriesName === undefined) { seriesName = yName; } From 584c0d8509e17f5c710ea9ec64eae54202219d00 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 5 Nov 2013 13:50:27 +0200 Subject: [PATCH 030/540] Support multiple series on one row --- rd_ui/app/scripts/services.js | 38 ++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 61be9b940..136a4665a 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -87,21 +87,20 @@ _.each(this.getData(), function (row) { var point = {}; var seriesName = undefined; - var yName = ""; - var xName = ""; + var xValue = 0; + var yValues = {}; - - _.map(row, function (value, definition) { + _.each(row, function (value, definition) { var type = definition.split("::")[1]; var name = definition.split("::")[0]; if (type == 'x') { - xName = name; + xValue = value; point[type] = value; } if (type == 'y') { - yName = name; + yValues[name] = value; point[type] = value; } @@ -110,19 +109,26 @@ } }); - if (seriesName === undefined) { - seriesName = yName; - } - - if (series[seriesName] == undefined) { - series[seriesName] = { - name: seriesName, - type: 'column', - data: [] + var addPointToSeries = function(seriesName, point) { + if (series[seriesName] == undefined) { + series[seriesName] = { + name: seriesName, + type: 'column', + data: [] + } } + + series[seriesName]['data'].push(point); + } + + if (seriesName === undefined) { + _.each(yValues, function(yValue, seriesName) { + addPointToSeries(seriesName, {'x': xValue, 'y': yValue}); + }); + } else { + addPointToSeries(seriesName, point); } - series[seriesName]['data'].push(point); }); return _.values(series); From 5d3e40130288708d0bd5263a0d52e191dcfeea3d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 6 Nov 2013 15:03:08 +0200 Subject: [PATCH 031/540] Show shared tooltip & percentage --- rd_ui/app/scripts/query_fiddle/renderers.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 2878850a8..9b085e072 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -4,7 +4,24 @@ var defaultChartOptions = { "text": null }, "tooltip": { - valueDecimals: 2 + valueDecimals: 2, + formatter: function () { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; + + + $.each(this.points, function (i, point) { + s += '
    ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; + } + }); + + return s; + }, + shared: true }, xAxis: { type: 'datetime' From 340284e09b62e36a4b095e21817fb4edc96fa125 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 6 Nov 2013 15:24:41 +0200 Subject: [PATCH 032/540] Fixes #38: getChartData returns sorted data by x --- rd_ui/app/scripts/services.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 136a4665a..13ad4a7aa 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -128,7 +128,10 @@ } else { addPointToSeries(seriesName, point); } + }); + _.each(series, function(series) { + series.data = _.sortBy(series.data, 'x'); }); return _.values(series); From b88dbb1a6f59342f97be7c1ed1e90bcbdf9230e4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 17 Nov 2013 13:41:45 +0200 Subject: [PATCH 033/540] Add support for displaying cohorts in dashbaords --- rd_ui/app/scripts/directives.js | 2 +- rd_ui/app/views/dashboard.html | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a2149954e..313f1944f 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -112,7 +112,7 @@ directives.directive('newWidgetForm', ['$http', function($http) { templateUrl: '/views/new_widget_form.html', replace: true, link: function($scope, element, attrs) { - $scope.widgetTypes = [{name: 'Chart', value: 'chart'}, {name: 'Table', value: 'grid'}]; + $scope.widgetTypes = [{name: 'Chart', value: 'chart'}, {name: 'Table', value: 'grid'}, {name: 'Cohort', value: 'cohort'}]; $scope.widgetSizes = [{name: 'Regular Size', value: 1}, {name: 'Double Size', value: 2}]; var reset = function() { diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 30f2b4078..b16ac3ea3 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -37,6 +37,7 @@
    +
    From 0a7949540cd704c33a3dfcf1449108bcd41e8836 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 24 Nov 2013 16:06:01 +0200 Subject: [PATCH 034/540] Fix: use String ctor function instead of toString to handle nulls --- rd_ui/app/scripts/services.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 13ad4a7aa..18d5df11b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -105,7 +105,7 @@ } if (type == 'series') { - seriesName = value.toString(); + seriesName = value ? value.toString() : "null"; } }); From 0f89a12a3d7d2185452e6482319056ebbcbaba39 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 24 Nov 2013 16:06:01 +0200 Subject: [PATCH 035/540] Fix: use String ctor function instead of toString to handle nulls --- rd_ui/app/scripts/services.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 18d5df11b..47b3a75e2 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -105,7 +105,7 @@ } if (type == 'series') { - seriesName = value ? value.toString() : "null"; + seriesName = String(value); } }); From 24492bbe2d8f8315085dedca3aa58bb689d4f5ef Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 20:40:26 +0200 Subject: [PATCH 036/540] Log exception when refresh queries fails --- rd_service/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rd_service/cli.py b/rd_service/cli.py index fae1c0a21..be6112717 100644 --- a/rd_service/cli.py +++ b/rd_service/cli.py @@ -21,8 +21,9 @@ def start_workers(data_manager): while True: try: data_manager.refresh_queries() - except Exception: - logging.error("Something went wrong with refreshing queries..."); + except Exception as e: + logging.error("Something went wrong with refreshing queries...") + logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") @@ -32,6 +33,7 @@ def start_workers(data_manager): if __name__ == '__main__': channel = logging.StreamHandler() logging.getLogger().addHandler(channel) + # TODO: take logging level from configuration logging.getLogger().setLevel("DEBUG") parser = argparse.ArgumentParser() From 1131df9c7844a2ec5107a5b2d8ee3a3de5a224da Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 20:40:44 +0200 Subject: [PATCH 037/540] Run query in forked process --- rd_service/data/manager.py | 2 +- rd_service/data/worker.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 3af229f30..bd4ab9acd 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -147,7 +147,7 @@ class Manager(object): pg_connection_pool = psycopg2.pool.ThreadedConnectionPool(1, max_connections, connection_string) runner = query_runner.redshift(pg_connection_pool) - self.workers = [worker.Worker(self, runner) for i in range(workers_count)] + self.workers = [worker.Worker(self, runner) for _ in range(workers_count)] for w in self.workers: w.start() diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 97ad419e6..ff3833dfb 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -3,6 +3,7 @@ Worker implementation to execute incoming queries. """ import json import logging +import os import threading import uuid import datetime @@ -135,11 +136,21 @@ class Worker(threading.Thread): job_id = self.manager.queue.pop() if job_id: logging.info("[%s] Processing %s", self.name, job_id) - self._process(job_id) - logging.info("[%s] Finished Processing %s", self.name, job_id) + + self._fork_and_process(job_id) else: time.sleep(self.sleep_time) + def _fork_and_process(self, job_id): + child_pid = os.fork() + if child_pid == 0: + self._process(job_id) + else: + logging.info("[%s] Waiting for pid: %d", self.name, child_pid) + _, status = os.waitpid(child_pid, 0) + logging.info("[%s] Finished Processing %s (pid: %d status: %d)", + self.name, job_id, child_pid, status) + def _process(self, job_id): job = Job.load(self.manager, job_id) if job.is_finished(): From a88582a57986182abb2b976abe1eafd607efa8cc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 20:53:51 +0200 Subject: [PATCH 038/540] Fix: child process wasn't exiting --- rd_service/data/worker.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index ff3833dfb..801468632 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -127,6 +127,7 @@ class Worker(threading.Thread): self.continue_working = True self.query_runner = query_runner self.sleep_time = sleep_time + self.child_pid = None super(Worker, self).__init__(name="Worker-%s" % uuid.uuid1()) @@ -136,20 +137,21 @@ class Worker(threading.Thread): job_id = self.manager.queue.pop() if job_id: logging.info("[%s] Processing %s", self.name, job_id) - self._fork_and_process(job_id) + if self.child_pid == 0: + return else: time.sleep(self.sleep_time) def _fork_and_process(self, job_id): - child_pid = os.fork() - if child_pid == 0: + self.child_pid = os.fork() + if self.child_pid == 0: self._process(job_id) else: - logging.info("[%s] Waiting for pid: %d", self.name, child_pid) - _, status = os.waitpid(child_pid, 0) + logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) + _, status = os.waitpid(self.child_pid, 0) logging.info("[%s] Finished Processing %s (pid: %d status: %d)", - self.name, job_id, child_pid, status) + self.name, job_id, self.child_pid, status) def _process(self, job_id): job = Job.load(self.manager, job_id) From 7ef1ed400ba56cb10e58e5e24569e241c08f9e19 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 20:54:41 +0200 Subject: [PATCH 039/540] Don't use connection pool in query runner --- rd_service/cli.py | 3 +-- rd_service/data/manager.py | 6 ++---- rd_service/data/query_runner.py | 9 ++++----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/rd_service/cli.py b/rd_service/cli.py index be6112717..b706c7acb 100644 --- a/rd_service/cli.py +++ b/rd_service/cli.py @@ -14,8 +14,7 @@ import data def start_workers(data_manager): try: - data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_STRING, - settings.MAX_CONNECTIONS) + data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_STRING) logging.info("Workers started.") while True: diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index bd4ab9acd..825b7ede2 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -139,13 +139,11 @@ class Manager(object): return data - def start_workers(self, workers_count, connection_string, max_connections): + def start_workers(self, workers_count, connection_string): if self.workers: return self.workers - # TODO: who closes the connection pool? - pg_connection_pool = psycopg2.pool.ThreadedConnectionPool(1, max_connections, connection_string) - runner = query_runner.redshift(pg_connection_pool) + runner = query_runner.redshift(connection_string) self.workers = [worker.Worker(self, runner) for _ in range(workers_count)] for w in self.workers: diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index 5ea4d1a1d..2aeaa20d2 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -12,12 +12,12 @@ import sys from .utils import JSONEncoder -def redshift(connection_pool): +def redshift(connection_string): def column_friendly_name(column_name): return column_name def query_runner(query): - connection = connection_pool.getconn() + connection = psycopg2.connect(connection_string) cursor = connection.cursor() try: @@ -41,10 +41,9 @@ def redshift(connection_pool): except Exception as e: connection.rollback() - connection_pool.putconn(connection) raise sys.exc_info()[1], None, sys.exc_info()[2] - - connection_pool.putconn(connection) + finally: + connection.close() return json_data, error From 8381bd14c57afcb6e702c85c2ded035af588314f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 20:59:48 +0200 Subject: [PATCH 040/540] If worker forked process didn't exit cleanly, update job --- rd_service/data/worker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 801468632..7cf90dfc4 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -150,6 +150,13 @@ class Worker(threading.Thread): else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) _, status = os.waitpid(self.child_pid, 0) + if status > 0: + job = Job.load(self.manager, job_id) + if not job.is_finished(): + logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", + self.name, job_id) + job.done(None, "Interrupted/Cancelled while running.") + logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) From a604fcf8a4aa7c7cc0b9947e3ee59eee22906ae1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 21:04:22 +0200 Subject: [PATCH 041/540] Store job process id --- rd_service/data/worker.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 7cf90dfc4..4b2c3087b 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -23,12 +23,14 @@ class Job(object): def __init__(self, data_manager, query, priority, job_id=None, wait_time=None, query_time=None, - updated_at=None, status=None, error=None, query_result_id=None): + updated_at=None, status=None, error=None, query_result_id=None, + process_id=0): self.data_manager = data_manager self.query = query self.priority = priority self.query_hash = gen_query_hash(self.query) self.query_result_id = query_result_id + self.process_id = process_id if job_id is None: self.id = str(uuid.uuid1()) @@ -57,7 +59,8 @@ class Job(object): 'updated_at': self.updated_at, 'status': self.status, 'error': self.error, - 'query_result_id': self.query_result_id + 'query_result_id': self.query_result_id, + 'process_id': self.process_id } @staticmethod @@ -79,8 +82,9 @@ class Job(object): pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict())) pipe.execute() - def processing(self): + def processing(self, process_id): self.status = self.PROCESSING + self.process_id = process_id self.wait_time = time.time() - self.updated_at self.updated_at = time.time() self.save() @@ -116,7 +120,8 @@ class Job(object): priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']), status=int(job_dict['status']), wait_time=float(job_dict['wait_time']), query_time=float(job_dict['query_time']), error=job_dict['error'], - query_result_id=job_dict['query_result_id']) + query_result_id=job_dict['query_result_id'], + process_id=int(job_dict['process_id'])) return job @@ -166,7 +171,7 @@ class Worker(threading.Thread): logging.warning("[%s][%s] tried to process finished job.", self.name, job) return - job.processing() + job.processing(os.getpid()) logging.info("[%s][%s] running query...", self.name, job.id) start_time = time.time() From e8aa8e094f21d0902e0e26ad10174b5adae0ba21 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 21:09:24 +0200 Subject: [PATCH 042/540] Naive implementation of job cancel --- rd_service/data/worker.py | 12 ++++++++++++ rd_service/server.py | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 4b2c3087b..7347f1b07 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -8,6 +8,7 @@ import threading import uuid import datetime import time +import signal from utils import gen_query_hash @@ -67,6 +68,17 @@ class Job(object): def _redis_key(job_id): return 'job:%s' % job_id + def cancel(self): + # TODO: Race condition: + # it's possible that it will be picked up by worker while processing the cancel order + if self.is_finished(): + return + + if self.status == self.PROCESSING: + os.kill(self.process_id, signal.SIGINT) + else: + self.done(None, "Interrupted/Cancelled while running.") + def save(self, pipe=None): if not pipe: pipe = self.data_manager.redis_connection.pipeline() diff --git a/rd_service/server.py b/rd_service/server.py index 738b3e1f7..5bfcd4ac6 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -251,8 +251,9 @@ class JobsHandler(BaseHandler): raise NotImplemented def delete(self, job_id): - raise NotImplemented - + job = data.Job.load(self.data_manager, job_id) + job.cancel() + class CsvQueryResultsHandler(BaseHandler): def get(self, query_result_id): From 1e3be5b4b87bb7c3857a27ff36d443631042dcfd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 21:27:50 +0200 Subject: [PATCH 043/540] Job: support for old job that had no process id. --- rd_service/data/worker.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 7347f1b07..6d0cb4f9b 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -31,7 +31,10 @@ class Job(object): self.priority = priority self.query_hash = gen_query_hash(self.query) self.query_result_id = query_result_id - self.process_id = process_id + if process_id == 'None': + self.process_id = None + else: + self.process_id = int(process_id) if job_id is None: self.id = str(uuid.uuid1()) @@ -133,7 +136,7 @@ class Job(object): status=int(job_dict['status']), wait_time=float(job_dict['wait_time']), query_time=float(job_dict['query_time']), error=job_dict['error'], query_result_id=job_dict['query_result_id'], - process_id=int(job_dict['process_id'])) + process_id=job_dict['process_id']) return job From e47f78f657f5291c99287a9eb4321202e98a07de Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 21:28:04 +0200 Subject: [PATCH 044/540] Job: when cancelling send SIGKILL instead of SIGINT. --- rd_service/data/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 6d0cb4f9b..d4a551fb0 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -78,7 +78,7 @@ class Job(object): return if self.status == self.PROCESSING: - os.kill(self.process_id, signal.SIGINT) + os.kill(self.process_id, signal.SIGKILL) else: self.done(None, "Interrupted/Cancelled while running.") From 206709b703caa7171532eb9bb3904cb3c359031b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 29 Nov 2013 21:28:20 +0200 Subject: [PATCH 045/540] Show cancel button in UI. --- rd_service/server.py | 2 +- rd_ui/app/scripts/controllers.js | 7 +++++++ rd_ui/app/scripts/services.js | 4 ++++ rd_ui/app/views/queryfiddle.html | 10 ++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index 5bfcd4ac6..d1463074c 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -253,7 +253,7 @@ class JobsHandler(BaseHandler): def delete(self, job_id): job = data.Job.load(self.data_manager, job_id) job.cancel() - + class CsvQueryResultsHandler(BaseHandler): def get(self, query_result_id): diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 5fa5136d8..e13af8fcb 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -154,7 +154,14 @@ $scope.executeQuery = function() { $scope.queryResult = $scope.query.getQueryResult(0); $scope.lockButton(true); + $scope.cancelling = false; } + + $scope.cancelExecution = function() { + $scope.cancelling = true; + $scope.queryResult.cancelExecution(); + } + } var QueriesCtrl = function($scope, $http, $location, Query) { diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 47b3a75e2..847e6afaa 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -51,6 +51,10 @@ return id; } + QueryResult.prototype.cancelExecution = function() { + Job.delete({id: this.job.id}); + } + QueryResult.prototype.getStatus = function() { return this.status || statuses[this.job.status]; } diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index cd44030bc..3da0f4082 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -34,8 +34,14 @@
    -
    Executing query...
    -
    Query in queue...
    +
    + Executing query... + +
    +
    + Query in queue... + +
    Error running query: {{queryResult.getError()}}
    From cd03948164934858692d16622daaf74f355c789f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 1 Dec 2013 11:52:14 +0200 Subject: [PATCH 046/540] Instead of killing the process, send SIGINT and cancel query on interupt. --- rd_service/data/query_runner.py | 26 ++++++++++++++++++++++---- rd_service/data/worker.py | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index 2aeaa20d2..29631ce39 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -9,6 +9,8 @@ query language (for example: HiveQL). import json import psycopg2 import sys +import signal +import select from .utils import JSONEncoder @@ -16,12 +18,27 @@ def redshift(connection_string): def column_friendly_name(column_name): return column_name + def wait(conn): + while 1: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + def query_runner(query): - connection = psycopg2.connect(connection_string) + connection = psycopg2.connect(connection_string, async=True) + wait(connection) + cursor = connection.cursor() try: cursor.execute(query) + wait(connection) column_names = [col.name for col in cursor.description] @@ -35,12 +52,13 @@ def redshift(connection_string): error = None cursor.close() except psycopg2.DatabaseError as e: - connection.rollback() json_data = None error = e.message - + except KeyboardInterrupt: + connection.cancel() + error = "Query cancelled by user." + json_data = None except Exception as e: - connection.rollback() raise sys.exc_info()[1], None, sys.exc_info()[2] finally: connection.close() diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index d4a551fb0..6d0cb4f9b 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -78,7 +78,7 @@ class Job(object): return if self.status == self.PROCESSING: - os.kill(self.process_id, signal.SIGKILL) + os.kill(self.process_id, signal.SIGINT) else: self.done(None, "Interrupted/Cancelled while running.") From 25c6bc2252c0e897c35b91e8b061e531e3db1a67 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 6 Dec 2013 15:14:39 +0200 Subject: [PATCH 047/540] Improved system status. --- rd_service/cli.py | 8 ++++++-- rd_service/data/manager.py | 17 ++++++++++++++++ rd_service/data/query_runner.py | 1 - rd_service/data/worker.py | 27 +++++++++++++++++++++++++- rd_service/server.py | 21 +++++++++++++++++++- rd_ui/app/scripts/admin_controllers.js | 4 ++++ rd_ui/app/views/admin_status.html | 16 +++++++++++++++ 7 files changed, 89 insertions(+), 5 deletions(-) diff --git a/rd_service/cli.py b/rd_service/cli.py index b706c7acb..36648bd06 100644 --- a/rd_service/cli.py +++ b/rd_service/cli.py @@ -14,6 +14,11 @@ import data def start_workers(data_manager): try: + old_workers = data_manager.redis_connection.smembers('workers') + data_manager.redis_connection.delete('workers') + + logging.info("Cleaning old workers: %s", old_workers) + data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_STRING) logging.info("Workers started.") @@ -32,8 +37,7 @@ def start_workers(data_manager): if __name__ == '__main__': channel = logging.StreamHandler() logging.getLogger().addHandler(channel) - # TODO: take logging level from configuration - logging.getLogger().setLevel("DEBUG") + logging.getLogger().setLevel(settings.LOG_LEVEL) parser = argparse.ArgumentParser() parser.add_argument("command") diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 825b7ede2..f307c4967 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -9,6 +9,7 @@ import psycopg2 import psycopg2.pool import qr import redis +import time import query_runner import worker from utils import gen_query_hash @@ -32,6 +33,12 @@ class Manager(object): db_connection_string) self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 + self.status = { + 'last_refresh_at': 0, + 'started_at': time.time() + } + + self._save_status() # TODO: Use our Django Models def get_query_result_by_id(self, query_result_id): @@ -95,6 +102,8 @@ class Manager(object): return job def refresh_queries(self): + logging.info("Refreshing queries...") + sql = """SELECT queries.query, queries.ttl, retrieved_at FROM (SELECT query, min(ttl) as ttl FROM queries WHERE ttl > 0 GROUP by query) queries JOIN (SELECT query, max(retrieved_at) as retrieved_at @@ -103,10 +112,15 @@ class Manager(object): WHERE queries.ttl > 0 AND query_results.retrieved_at + ttl * interval '1 second' < now() at time zone 'utc';""" + self.status['last_refresh_at'] = time.time() + self._save_status() + queries = self.run_query(sql) for query, ttl, retrieved_at in queries: self.add_job(query, worker.Job.LOW_PRIORITY) + logging.info("Done refreshing queries... %d" % len(queries)) + def store_query_result(self, query, data, run_time, retrieved_at): query_result_id = None query_hash = gen_query_hash(query) @@ -169,3 +183,6 @@ class Manager(object): connection.commit() finally: self.db_connection_pool.putconn(connection) + + def _save_status(self): + self.redis_connection.hmset('manager:status', self.status) diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index 29631ce39..a39fff8c6 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -9,7 +9,6 @@ query language (for example: HiveQL). import json import psycopg2 import sys -import signal import select from .utils import JSONEncoder diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 6d0cb4f9b..f7df3b022 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -148,14 +148,25 @@ class Worker(threading.Thread): self.query_runner = query_runner self.sleep_time = sleep_time self.child_pid = None + self.worker_id = uuid.uuid1() + self.status = { + 'jobs_count': 0, + 'cancelled_jobs_count': 0, + 'done_jobs_count': 0, + 'updated_at': time.time(), + 'started_at': time.time() + } + self._save_status() + self.manager.redis_connection.sadd('workers', self._key) - super(Worker, self).__init__(name="Worker-%s" % uuid.uuid1()) + super(Worker, self).__init__(name="Worker-%s" % self.worker_id) def run(self): logging.info("[%s] started.", self.name) while self.continue_working: job_id = self.manager.queue.pop() if job_id: + self._update_status('jobs_count') logging.info("[%s] Processing %s", self.name, job_id) self._fork_and_process(job_id) if self.child_pid == 0: @@ -163,6 +174,18 @@ class Worker(threading.Thread): else: time.sleep(self.sleep_time) + def _update_status(self, counter): + self.status['updated_at'] = time.time() + self.status[counter] += 1 + self._save_status() + + @property + def _key(self): + return 'worker:%s' % self.worker_id + + def _save_status(self): + self.manager.redis_connection.hmset(self._key, self.status) + def _fork_and_process(self, job_id): self.child_pid = os.fork() if self.child_pid == 0: @@ -171,8 +194,10 @@ class Worker(threading.Thread): logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) _, status = os.waitpid(self.child_pid, 0) if status > 0: + self._update_status('done_jobs_count') job = Job.load(self.manager, job_id) if not job.is_finished(): + self._update_status('cancelled_jobs_count') logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", self.name, job_id) job.done(None, "Interrupted/Cancelled while running.") diff --git a/rd_service/server.py b/rd_service/server.py index d1463074c..3ecf1f4a1 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -30,6 +30,7 @@ import tornado.web import tornado.auth import tornado.options import settings +import time from data import utils import data @@ -105,12 +106,30 @@ class StatusHandler(BaseHandler): status = {} info = self.redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] - status['queries_in_queue'] = self.redis_connection.zcard('jobs') + status['queries_count'] = data.models.Query.objects.count() status['query_results_count'] = data.models.QueryResult.objects.count() status['dashboards_count'] = data.models.Dashboard.objects.count() status['widgets_count'] = data.models.Widget.objects.count() + workers = [self.redis_connection.hgetall(w) + for w in self.redis_connection.smembers('workers')] + + status['workers'] = [] + for worker in workers: + w = {} + w['uptime'] = "%.0f seconds" % (time.time() - float(worker['started_at'])) + w['last_updated_at'] = datetime.datetime.fromtimestamp(float(worker['updated_at'])).strftime('%Y-%m-%d %H:%M:%S') + w['jobs_received'] = worker['jobs_count'] + w['jobs_done'] = worker['done_jobs_count'] + status['workers'].append(w) + + manager_status = self.redis_connection.hgetall('manager:status') + status['manager'] = {} + status['manager']['uptime'] = "%.0f seconds" % (time.time() - float(manager_status['started_at'])) + status['manager']['last_refresh_at'] = datetime.datetime.fromtimestamp(float(manager_status['last_refresh_at'])).strftime('%Y-%m-%d %H:%M:%S') + status['manager']['queue_size'] = self.redis_connection.zcard('jobs') + self.write_json(status) diff --git a/rd_ui/app/scripts/admin_controllers.js b/rd_ui/app/scripts/admin_controllers.js index 93311711d..f05c67caa 100644 --- a/rd_ui/app/scripts/admin_controllers.js +++ b/rd_ui/app/scripts/admin_controllers.js @@ -5,6 +5,10 @@ var refresh = function () { $scope.refresh_time = moment().add('minutes', 1); $http.get('/status.json').success(function (data) { + $scope.workers = data.workers; + delete data.workers; + $scope.manager = data.manager; + delete data.manager; $scope.status = data; }); diff --git a/rd_ui/app/views/admin_status.html b/rd_ui/app/views/admin_status.html index f2ff63d22..9e2ce64cb 100644 --- a/rd_ui/app/views/admin_status.html +++ b/rd_ui/app/views/admin_status.html @@ -10,6 +10,22 @@ {{name | toHuman}} +
      +
    • Manager
      • +
    • + {{value}} + {{name | toHuman}} +
      • +
    +
      +
      +
    • Worker {{$index+1}}
      • +
    • + {{value}} + {{name | toHuman}} +
      • +
      +
    Next refresh:
    From c10c7a959d9013edbdd7fb46228dd87f13fdfec7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 6 Dec 2013 15:32:36 +0200 Subject: [PATCH 048/540] Jobs done counter was updated in the wrong place --- rd_service/data/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index f7df3b022..e4d13aed7 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -193,8 +193,8 @@ class Worker(threading.Thread): else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) _, status = os.waitpid(self.child_pid, 0) + self._update_status('done_jobs_count') if status > 0: - self._update_status('done_jobs_count') job = Job.load(self.manager, job_id) if not job.is_finished(): self._update_status('cancelled_jobs_count') From fef4dadd58a9adef377475f6272a5f2097d43cfd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 6 Dec 2013 15:50:02 +0200 Subject: [PATCH 049/540] More readable status --- rd_service/server.py | 17 +++------------- rd_ui/app/views/admin_status.html | 34 ++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index 3ecf1f4a1..4e0e7a2a8 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -112,22 +112,11 @@ class StatusHandler(BaseHandler): status['dashboards_count'] = data.models.Dashboard.objects.count() status['widgets_count'] = data.models.Widget.objects.count() - workers = [self.redis_connection.hgetall(w) - for w in self.redis_connection.smembers('workers')] - - status['workers'] = [] - for worker in workers: - w = {} - w['uptime'] = "%.0f seconds" % (time.time() - float(worker['started_at'])) - w['last_updated_at'] = datetime.datetime.fromtimestamp(float(worker['updated_at'])).strftime('%Y-%m-%d %H:%M:%S') - w['jobs_received'] = worker['jobs_count'] - w['jobs_done'] = worker['done_jobs_count'] - status['workers'].append(w) + status['workers'] = [self.redis_connection.hgetall(w) + for w in self.redis_connection.smembers('workers')] manager_status = self.redis_connection.hgetall('manager:status') - status['manager'] = {} - status['manager']['uptime'] = "%.0f seconds" % (time.time() - float(manager_status['started_at'])) - status['manager']['last_refresh_at'] = datetime.datetime.fromtimestamp(float(manager_status['last_refresh_at'])).strftime('%Y-%m-%d %H:%M:%S') + status['manager'] = manager_status status['manager']['queue_size'] = self.redis_connection.zcard('jobs') self.write_json(status) diff --git a/rd_ui/app/views/admin_status.html b/rd_ui/app/views/admin_status.html index 9e2ce64cb..a08e6c3ac 100644 --- a/rd_ui/app/views/admin_status.html +++ b/rd_ui/app/views/admin_status.html @@ -12,18 +12,38 @@
    • Manager
      • -
    • - {{value}} - {{name | toHuman}} +
    • + + Last Refresh +
      • +
    • + + Started +
      • +
    • + {{manager.queue_size}} + Queue Size
    • Worker {{$index+1}}
      • -
    • - {{value}} - {{name | toHuman}} -
      • +
    • + + Updated +
      • +
    • + + Started +
      • +
    • + {{worker.jobs_count}} + Jobs Received +
      • +
    • + {{worker.done_jobs_count}} + Jobs Done +
    From fc7412adae79016b9654fd39b0ed8029246797a0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 6 Dec 2013 17:52:13 +0200 Subject: [PATCH 050/540] Add log_level to settings_example --- rd_service/settings_example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_service/settings_example.py b/rd_service/settings_example.py index da2e35409..af8c3d04e 100644 --- a/rd_service/settings_example.py +++ b/rd_service/settings_example.py @@ -21,6 +21,7 @@ STATIC_ASSETS_PATH = "../rd_ui/dist/" WORKERS_COUNT = 2 MAX_CONNECTIONS = 3 COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" +LOG_LEVEL = "INFO" # Configuration of the operational database for the Django models django.conf.settings.configure(DATABASES = { 'default': { From 7068e68b8fdb168294591a072bef3269e0780857 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 6 Dec 2013 20:56:08 +0200 Subject: [PATCH 051/540] No more using connection pool in DataManager, as it used accross processes --- rd_service/data/manager.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index f307c4967..81f7d46a2 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -6,7 +6,6 @@ from contextlib import contextmanager import json import logging import psycopg2 -import psycopg2.pool import qr import redis import time @@ -29,8 +28,7 @@ class Manager(object): def __init__(self, redis_connection, db_connection_string, db_max_connections): self.redis_connection = redis_connection self.workers = [] - self.db_connection_pool = psycopg2.pool.ThreadedConnectionPool(1, db_max_connections, - db_connection_string) + self.db_connection_string = db_connection_string self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 self.status = { @@ -102,8 +100,6 @@ class Manager(object): return job def refresh_queries(self): - logging.info("Refreshing queries...") - sql = """SELECT queries.query, queries.ttl, retrieved_at FROM (SELECT query, min(ttl) as ttl FROM queries WHERE ttl > 0 GROUP by query) queries JOIN (SELECT query, max(retrieved_at) as retrieved_at @@ -115,6 +111,7 @@ class Manager(object): self.status['last_refresh_at'] = time.time() self._save_status() + logging.info("Refreshing queries...") queries = self.run_query(sql) for query, ttl, retrieved_at in queries: self.add_job(query, worker.Job.LOW_PRIORITY) @@ -172,7 +169,7 @@ class Manager(object): @contextmanager def db_transaction(self): - connection = self.db_connection_pool.getconn() + connection = psycopg2.connect(self.db_connection_string) cursor = connection.cursor() try: yield cursor @@ -182,7 +179,7 @@ class Manager(object): else: connection.commit() finally: - self.db_connection_pool.putconn(connection) + connection.close() def _save_status(self): self.redis_connection.hmset('manager:status', self.status) From cc86cb5ffad1931e9dd94f5d9cc6f8942d0de765 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 7 Dec 2013 13:14:02 +0200 Subject: [PATCH 052/540] Support for non date time x values. --- rd_ui/app/scripts/ng-highchart.js | 14 +++++++-- rd_ui/app/scripts/query_fiddle/renderers.js | 33 +++++++++++---------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 00e83c5db..2dae23aec 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -37,13 +37,23 @@ angular.module('highchart', []) if (!length || length == 0) { scope.chart.showLoading(); } else { - while(scope.chart.series.length > 0) { scope.chart.series[0].remove(true); } - scope.chart.counters.color = 0; + if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { + scope.chart.xAxis[0].update({type: 'category'}); + _.each(scope.series, function(s) { + _.each(s.data, function(p) { + p.name = p.x; + delete p.x; + }) + }) + } else { + scope.chart.xAxis[0].update({type: 'datetime'}); + } + scope.chart.counters.color = 0; _.each(scope.series, function(s) { scope.chart.addSeries(s); diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 9b085e072..7c47a4b0e 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -6,18 +6,25 @@ var defaultChartOptions = { "tooltip": { valueDecimals: 2, formatter: function () { - var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', - pointsCount = this.points.length; + if (moment.isMoment(this.x)) { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; + $.each(this.points, function (i, point) { + s += '
    ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); - $.each(this.points, function (i, point) { - s += '
    ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - - if (pointsCount > 1 && point.percentage) { - s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; - } - }); + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; + } + }); + } else { + var s = "" + this.points[0].key + ""; + $.each(this.points, function (i, point) { + s+= '
    ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + }); + } return s; }, @@ -86,11 +93,7 @@ renderers.directive('chartRenderer', function () { $scope.chartOptions = defaultChartOptions; $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data) { - return; - } - - if ($scope.queryResult.getData() == null) { + if (!data || $scope.queryResult.getData() == null) { $scope.chartSeries.splice(0, $scope.chartSeries.length); } else { $scope.chartSeries.splice(0, $scope.chartSeries.length); From 7e73b307f00bfbda6634aed1290199f597132f4e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 7 Dec 2013 14:18:39 +0200 Subject: [PATCH 053/540] Bring back browser notifications (#1) --- rd_ui/app/scripts/controllers.js | 7 +++++-- rd_ui/app/scripts/query_fiddle/renderers.js | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index e13af8fcb..cb8d7f992 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -29,7 +29,7 @@ $scope.updateTime = ''; } - var QueryFiddleCtrl = function ($scope, $routeParams, $http, $location, growl, Query) { + var QueryFiddleCtrl = function ($scope, $routeParams, $http, $location, growl, notifications, Query) { $scope.$parent.pageTitle = "Query Fiddle"; $scope.tabs = [{'key': 'table', 'name': 'Table'}, {'key': 'chart', 'name': 'Chart'}, @@ -134,6 +134,9 @@ Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()}) } $scope.query.latest_query_data_id = $scope.queryResult.getId(); + + notifications.showNotification("re:dash", $scope.query.name + " updated."); + $scope.lockButton(false); } }); @@ -236,7 +239,7 @@ .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', 'Query', QueriesCtrl]) - .controller('QueryFiddleCtrl', ['$scope', '$routeParams', '$http', '$location', 'growl', 'Query', QueryFiddleCtrl]) + .controller('QueryFiddleCtrl', ['$scope', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 7c47a4b0e..36caa05ca 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -106,9 +106,6 @@ renderers.directive('chartRenderer', function () { _.each($scope.queryResult.getChartData(), function (s) { $scope.chartSeries.push(_.extend(s, {'stacking': stacking})); }); - - // TODO: move this to the parent controller - // notifications.showNotification("RedDash", $scope.query.name + " updated."); } }); }] From 1e92240cbdd4971bea3307df057683269eee6e07 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 8 Dec 2013 15:25:44 +0200 Subject: [PATCH 054/540] Return query stats (runtime and such) in API. --- rd_service/data/models.py | 26 +++++++++++++++++++++++++- rd_service/server.py | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index c49e0f195..9559e8ad6 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -48,7 +48,7 @@ class Query(models.Model): app_label = 'redash' db_table = 'queries' - def to_dict(self, with_result=True): + def to_dict(self, with_result=True, with_stats=False): d = { 'id': self.id, 'latest_query_data_id': self.latest_query_data_id, @@ -61,11 +61,35 @@ class Query(models.Model): 'created_at': self.created_at, } + if with_stats: + d['avg_runtime'] = self.avg_runtime + d['min_runtime'] = self.min_runtime + d['max_runtime'] = self.max_runtime + d['last_retrieved_at'] = self.last_retrieved_at + d['times_retrieved'] = self.times_retrieved + if with_result and self.latest_query_data_id: d['latest_query_data'] = self.latest_query_data.to_dict() return d + @classmethod + def all_queries(cls): + query = """SELECT queries.*, query_stats.* +FROM queries +LEFT OUTER JOIN + (SELECT qu.query_hash, + count(0) AS "times_retrieved", + avg(runtime) AS "avg_runtime", + min(runtime) AS "min_runtime", + max(runtime) AS "max_runtime", + max(retrieved_at) AS "last_retrieved_at" + FROM queries qu + JOIN query_results qr ON qu.query_hash=qr.query_hash + GROUP BY qu.query_hash) query_stats ON query_stats.query_hash = queries.query_hash + """ + return cls.objects.raw(query) + def save(self, *args, **kwargs): self.query_hash = utils.gen_query_hash(self.query) super(Query, self).save(*args, **kwargs) diff --git a/rd_service/server.py b/rd_service/server.py index 4e0e7a2a8..f74837bd7 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -223,7 +223,7 @@ class QueriesHandler(BaseHandler): else: self.send_error(404) else: - self.write_json([q.to_dict(with_result=False) for q in data.models.Query.objects.all()]) + self.write_json([q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()]) class QueryResultsHandler(BaseHandler): From 668c60e1e8a24edc84f2e79e4ba318d51f6e1d2b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 8 Dec 2013 15:26:35 +0200 Subject: [PATCH 055/540] Improvements to queries page: (#6) 1. Search (client side). 2. Stats about queries. 3. Pagination. --- rd_ui/app/scripts/controllers.js | 112 ++++++++++++++++--- rd_ui/app/scripts/filters.js | 36 ++++-- rd_ui/app/views/queries.html | 31 +---- rd_ui/app/views/queries_query_name_cell.html | 1 + 4 files changed, 123 insertions(+), 57 deletions(-) create mode 100644 rd_ui/app/views/queries_query_name_cell.html diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index cb8d7f992..4ea351c12 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -167,31 +167,109 @@ } - var QueriesCtrl = function($scope, $http, $location, Query) { + var QueriesCtrl = function($scope, $http, $location, $filter, Query) { $scope.$parent.pageTitle = "All Queries"; - $scope.queries = Query.query(); + var dateFormatter = function (value) { + if (!value) return "-"; + return value.format("DD/MM/YY HH:mm"); + } + + var filterQueries = function() { + $scope.queries = _.filter($scope.allQueries, function(query) { + if (!$scope.selectedTab) { + return false; + } + + if ($scope.selectedTab.key == 'my') { + return query.user == currentUser.name && query.name != 'New Query'; + } else if ($scope.selectedTab.key == 'drafts') { + return query.user == currentUser.name && query.name == 'New Query'; + } + + return query.name != 'New Query'; + }); + } + + $scope.gridConfig = { + isPaginationEnabled: true, + itemsByPage: 50, + maxSize: 8, + isGlobalSearchActivated: true + } + + $scope.allQueries = []; + $scope.queries = []; + Query.query(function(queries) { + $scope.allQueries = _.map(queries, function(query) { + query.created_at = moment(query.created_at); + query.last_retrieved_at = moment(query.last_retrieved_at); + return query; + }); + + filterQueries(); + }); + + $scope.gridColumns = [ + { + "label": "Name", + "map": "name", + "cellTemplateUrl": "/views/queries_query_name_cell.html" + }, + { + 'label': 'Created By', + 'map': 'user' + }, + { + 'label': 'Created At', + 'map': 'created_at', + 'formatFunction': dateFormatter + }, + { + 'label': 'Runtime (avg)', + 'map': 'avg_runtime', + 'formatFunction': 'number', + 'formatParameter': 2 + }, + { + 'label': 'Runtime (min)', + 'map': 'min_runtime', + 'formatFunction': 'number', + 'formatParameter': 2 + }, + { + 'label': 'Runtime (max)', + 'map': 'max_runtime', + 'formatFunction': 'number', + 'formatParameter': 2 + }, + { + 'label': 'Last Executed At', + 'map': 'last_retrieved_at', + 'formatFunction': dateFormatter + }, + { + 'label': 'Times Executed', + 'map': 'times_retrieved' + }, + { + 'label': 'Update Schedule', + 'map': 'ttl', + 'formatFunction': function(value) { + return $filter('refreshRateHumanize')(value); + } + } +// id: 672, + ] $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}]; $scope.$watch('selectedTab', function(tab) { if (tab) { $scope.$parent.pageTitle = tab.name; } + + filterQueries(); }) - - $scope.filterQueries = function(query) { - if (!$scope.selectedTab) { - return false; - } - - if ($scope.selectedTab.key == 'my') { - return query.user == currentUser.name && query.name != 'New Query'; - } else if ($scope.selectedTab.key == 'drafts') { - return query.user == currentUser.name && query.name == 'New Query'; - } - - return query.name != 'New Query'; - } } var MainCtrl = function ($scope, Dashboard, notifications) { @@ -238,7 +316,7 @@ angular.module('redash.controllers', []) .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', 'Query', WidgetCtrl]) - .controller('QueriesCtrl', ['$scope', '$http', '$location', 'Query', QueriesCtrl]) + .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) .controller('QueryFiddleCtrl', ['$scope', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); diff --git a/rd_ui/app/scripts/filters.js b/rd_ui/app/scripts/filters.js index 629ded993..1ef898510 100644 --- a/rd_ui/app/scripts/filters.js +++ b/rd_ui/app/scripts/filters.js @@ -1,19 +1,31 @@ +var durationHumanize = function (duration) { + var humanized = ""; + if (duration == undefined) { + humanized = "-"; + } else if (duration < 60) { + humanized = Math.round(duration) + "s"; + } else if (duration >= 3600) { + var hours = Math.round(parseFloat(duration) / 60.0 / 60.0) + humanized = hours + "h"; + } else { + var minutes = Math.round(parseFloat(duration) / 60.0); + humanized = minutes + "m"; + } + return humanized; +} + angular.module('redash.filters', []). filter('durationHumanize', function () { - return function (duration) { - var humanized = ""; - if (duration == undefined) { - humanized = "-"; - } else if (duration < 60) { - humanized = Math.round(duration) + "s"; - } else if (duration >= 3600) { - var hours = Math.round(parseFloat(duration) / 60.0 / 60.0) - humanized = hours + "h"; + return durationHumanize; + }) + + .filter('refreshRateHumanize', function () { + return function (ttl) { + if (ttl==-1) { + return "Never"; } else { - var minutes = Math.round(parseFloat(duration) / 60.0); - humanized = minutes + "m"; + return "Every " + durationHumanize(ttl); } - return humanized; } }) diff --git a/rd_ui/app/views/queries.html b/rd_ui/app/views/queries.html index 2ae893e42..ee24cb80d 100644 --- a/rd_ui/app/views/queries.html +++ b/rd_ui/app/views/queries.html @@ -1,31 +1,6 @@
    - - - - - - - - - - - - - - - - - - - - -
    NameRuntimeCreated atUpdate ScheduleCreated By
    {{query.name}}{{query.runtime | durationHumanize}} - - Never - - Every {{query.ttl | durationHumanize}} - Manual - {{query.user}}
    - +
    diff --git a/rd_ui/app/views/queries_query_name_cell.html b/rd_ui/app/views/queries_query_name_cell.html new file mode 100644 index 000000000..2685e458d --- /dev/null +++ b/rd_ui/app/views/queries_query_name_cell.html @@ -0,0 +1 @@ +{{dataRow.name}} \ No newline at end of file From 5075795704e4052d002bf850f23f6424175d5c26 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 8 Dec 2013 15:44:46 +0200 Subject: [PATCH 056/540] Humanize query runtime --- rd_ui/app/scripts/controllers.js | 33 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 4ea351c12..5468875eb 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -169,6 +169,15 @@ var QueriesCtrl = function($scope, $http, $location, $filter, Query) { $scope.$parent.pageTitle = "All Queries"; + $scope.gridConfig = { + isPaginationEnabled: true, + itemsByPage: 50, + maxSize: 8, + isGlobalSearchActivated: true + } + + $scope.allQueries = []; + $scope.queries = []; var dateFormatter = function (value) { if (!value) return "-"; @@ -191,15 +200,6 @@ }); } - $scope.gridConfig = { - isPaginationEnabled: true, - itemsByPage: 50, - maxSize: 8, - isGlobalSearchActivated: true - } - - $scope.allQueries = []; - $scope.queries = []; Query.query(function(queries) { $scope.allQueries = _.map(queries, function(query) { query.created_at = moment(query.created_at); @@ -228,20 +228,23 @@ { 'label': 'Runtime (avg)', 'map': 'avg_runtime', - 'formatFunction': 'number', - 'formatParameter': 2 + 'formatFunction': function(value) { + return $filter('durationHumanize')(value); + } }, { 'label': 'Runtime (min)', 'map': 'min_runtime', - 'formatFunction': 'number', - 'formatParameter': 2 + 'formatFunction': function(value) { + return $filter('durationHumanize')(value); + } }, { 'label': 'Runtime (max)', 'map': 'max_runtime', - 'formatFunction': 'number', - 'formatParameter': 2 + 'formatFunction': function(value) { + return $filter('durationHumanize')(value); + } }, { 'label': 'Last Executed At', From c50639d13714f2d5f0590589916d0772cb10dfd9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 13 Dec 2013 18:46:14 +0200 Subject: [PATCH 057/540] Add support for refreshing a query once a week --- rd_ui/app/scripts/controllers.js | 2 +- rd_ui/app/scripts/filters.js | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 5468875eb..88bc53478 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -98,7 +98,7 @@ }) $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'}); - + $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'}); $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) { if (newError == undefined) { diff --git a/rd_ui/app/scripts/filters.js b/rd_ui/app/scripts/filters.js index 1ef898510..baf5618d6 100644 --- a/rd_ui/app/scripts/filters.js +++ b/rd_ui/app/scripts/filters.js @@ -4,8 +4,11 @@ var durationHumanize = function (duration) { humanized = "-"; } else if (duration < 60) { humanized = Math.round(duration) + "s"; + } else if (duration > 3600*24) { + var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0); + humanized = days + "days"; } else if (duration >= 3600) { - var hours = Math.round(parseFloat(duration) / 60.0 / 60.0) + var hours = Math.round(parseFloat(duration) / 60.0 / 60.0); humanized = hours + "h"; } else { var minutes = Math.round(parseFloat(duration) / 60.0); From 0f09bbc25332dd995b46f8a2da1064f7eb91737a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Dec 2013 18:52:30 +0200 Subject: [PATCH 058/540] Set proctitle for worker --- rd_service/data/worker.py | 14 ++++++++++++++ rd_service/requirements.txt | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index e4d13aed7..b8117b820 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -9,6 +9,7 @@ import uuid import datetime import time import signal +import setproctitle from utils import gen_query_hash @@ -161,6 +162,15 @@ class Worker(threading.Thread): super(Worker, self).__init__(name="Worker-%s" % self.worker_id) + def set_title(self, title=None): + base_title = "redash worker:%s" % self.worker_id + if title: + full_title = "%s - %s" % (base_title, title) + else: + full_title = base_title + + setproctitle.setproctitle(full_title) + def run(self): logging.info("[%s] started.", self.name) while self.continue_working: @@ -189,6 +199,7 @@ class Worker(threading.Thread): def _fork_and_process(self, job_id): self.child_pid = os.fork() if self.child_pid == 0: + self.set_title("processing %s" % job_id) self._process(job_id) else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) @@ -215,6 +226,7 @@ class Worker(threading.Thread): logging.info("[%s][%s] running query...", self.name, job.id) start_time = time.time() + self.set_title("running query %s" % job_id) data, error = self.query_runner(job.query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", @@ -224,7 +236,9 @@ class Worker(threading.Thread): # while we already marked the job as done query_result_id = None if not error: + self.set_title("storing results %s" % job_id) query_result_id = self.manager.store_query_result(job.query, data, run_time, datetime.datetime.utcnow()) + self.set_title("marking job as done %s" % job_id) job.done(query_result_id, error) diff --git a/rd_service/requirements.txt b/rd_service/requirements.txt index 83351077f..e38d6691d 100644 --- a/rd_service/requirements.txt +++ b/rd_service/requirements.txt @@ -5,4 +5,5 @@ sqlparse==0.1.8 Django==1.5.4 django-db-pool==0.0.10 qr==0.6.0 -python-dateutil==2.1 \ No newline at end of file +python-dateutil==2.1 +setproctitle==1.1.7 \ No newline at end of file From 6e0d1c613c5998cb19ce5b3bb74e192bb073b4bf Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 17 Dec 2013 12:02:58 +0200 Subject: [PATCH 059/540] Change Job to use redis connection instead of data manager & use own redis connection in forked process. --- rd_service/data/manager.py | 4 +++- rd_service/data/worker.py | 25 ++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 81f7d46a2..1eabb7a1d 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -156,7 +156,9 @@ class Manager(object): runner = query_runner.redshift(connection_string) - self.workers = [worker.Worker(self, runner) for _ in range(workers_count)] + redis_connection_params = self.redis_connection.connection_pool.connection_kwargs + self.workers = [worker.Worker(self, redis_connection_params, runner) + for _ in range(workers_count)] for w in self.workers: w.start() diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index b8117b820..80e74ac81 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -10,6 +10,7 @@ import datetime import time import signal import setproctitle +import redis from utils import gen_query_hash @@ -22,12 +23,12 @@ class Job(object): DONE = 3 FAILED = 4 - def __init__(self, data_manager, query, priority, + def __init__(self, redis_connection, query, priority, job_id=None, wait_time=None, query_time=None, updated_at=None, status=None, error=None, query_result_id=None, process_id=0): - self.data_manager = data_manager + self.redis_connection = redis_connection self.query = query self.priority = priority self.query_hash = gen_query_hash(self.query) @@ -85,7 +86,7 @@ class Job(object): def save(self, pipe=None): if not pipe: - pipe = self.data_manager.redis_connection.pipeline() + pipe = self.redis_connection.pipeline() if self.new_job: pipe.set('query_hash_job:%s' % self.query_hash, self.id) @@ -124,15 +125,15 @@ class Job(object): return "" % (self.id, self.priority, self.status) @classmethod - def _load(cls, data_manager, job_id): - return data_manager.redis_connection.hgetall(cls._redis_key(job_id)) + def _load(cls, redis_connection, job_id): + return redis_connection.hgetall(cls._redis_key(job_id)) @classmethod - def load(cls, data_manager, job_id): - job_dict = cls._load(data_manager, job_id) + def load(cls, redis_connection, job_id): + job_dict = cls._load(redis_connection, job_id) job = None if job_dict: - job = Job(data_manager, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'), + job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'), priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']), status=int(job_dict['status']), wait_time=float(job_dict['wait_time']), query_time=float(job_dict['query_time']), error=job_dict['error'], @@ -143,8 +144,9 @@ class Job(object): class Worker(threading.Thread): - def __init__(self, manager, query_runner, sleep_time=0.1): + def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1): self.manager = manager + self.redis_connection_params = redis_connection_params self.continue_working = True self.query_runner = query_runner self.sleep_time = sleep_time @@ -206,7 +208,7 @@ class Worker(threading.Thread): _, status = os.waitpid(self.child_pid, 0) self._update_status('done_jobs_count') if status > 0: - job = Job.load(self.manager, job_id) + job = Job.load(self.manager.redis_connection, job_id) if not job.is_finished(): self._update_status('cancelled_jobs_count') logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", @@ -217,7 +219,8 @@ class Worker(threading.Thread): self.name, job_id, self.child_pid, status) def _process(self, job_id): - job = Job.load(self.manager, job_id) + redis_connection = redis.StrictRedis(**self.redis_connection_params) + job = Job.load(redis_connection, job_id) if job.is_finished(): logging.warning("[%s][%s] tried to process finished job.", self.name, job) return From f08ebf22562feb684e81bd8b4fce850235129fa0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 17 Dec 2013 12:03:10 +0200 Subject: [PATCH 060/540] Run annotated queries. --- rd_service/data/worker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index 80e74ac81..f7998f7d2 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -225,12 +225,17 @@ class Worker(threading.Thread): logging.warning("[%s][%s] tried to process finished job.", self.name, job) return - job.processing(os.getpid()) + pid = os.getpid() + job.processing(pid) logging.info("[%s][%s] running query...", self.name, job.id) start_time = time.time() self.set_title("running query %s" % job_id) - data, error = self.query_runner(job.query) + + annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ + (pid, job.id, job.query_hash, job.priority, job.query) + + data, error = self.query_runner(annotated_query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", self.name, job.id, data and len(data), error) From eea0c2e060a7eaec3903b83a75ac8fec8eeef627 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 17 Dec 2013 12:19:50 +0200 Subject: [PATCH 061/540] More places where I need to use redis_connection --- rd_service/data/manager.py | 4 ++-- rd_service/server.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 1eabb7a1d..cf0760a87 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -82,9 +82,9 @@ class Manager(object): job_id = pipe.get('query_hash_job:%s' % query_hash) if job_id: logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id) - job = worker.Job.load(self, job_id) + job = worker.Job.load(self.redis_connectino, job_id) else: - job = worker.Job(self, query, priority) + job = worker.Job(self.redis_connectino, query, priority) pipe.multi() job.save(pipe) logging.info("[Manager][%s] Created new job: %s", query_hash, job.id) diff --git a/rd_service/server.py b/rd_service/server.py index f74837bd7..1c777e838 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -253,13 +253,13 @@ class JobsHandler(BaseHandler): def get(self, job_id=None): if job_id: # TODO: if finished, include the query result - job = data.Job.load(self.data_manager, job_id) + job = data.Job.load(self.data_manager.redis_connectino, job_id) self.write({'job': job.to_dict()}) else: raise NotImplemented def delete(self, job_id): - job = data.Job.load(self.data_manager, job_id) + job = data.Job.load(self.data_manager.redis_connectino, job_id) job.cancel() From 93162fed8587c5397d94f217028bc63798b9dbd1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 17 Dec 2013 12:46:06 +0200 Subject: [PATCH 062/540] Take only needed vals for connection params --- rd_service/data/manager.py | 4 ++-- rd_service/data/worker.py | 4 +++- rd_service/server.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index cf0760a87..e69e7290b 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -82,9 +82,9 @@ class Manager(object): job_id = pipe.get('query_hash_job:%s' % query_hash) if job_id: logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id) - job = worker.Job.load(self.redis_connectino, job_id) + job = worker.Job.load(self.redis_connection, job_id) else: - job = worker.Job(self.redis_connectino, query, priority) + job = worker.Job(self.redis_connection, query, priority) pipe.multi() job.save(pipe) logging.info("[Manager][%s] Created new job: %s", query_hash, job.id) diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index f7998f7d2..c89e8236d 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -146,7 +146,9 @@ class Job(object): class Worker(threading.Thread): def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1): self.manager = manager - self.redis_connection_params = redis_connection_params + + self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() + if k in ('host', 'db', 'password', 'port')} self.continue_working = True self.query_runner = query_runner self.sleep_time = sleep_time diff --git a/rd_service/server.py b/rd_service/server.py index 1c777e838..416f7514e 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -253,13 +253,13 @@ class JobsHandler(BaseHandler): def get(self, job_id=None): if job_id: # TODO: if finished, include the query result - job = data.Job.load(self.data_manager.redis_connectino, job_id) + job = data.Job.load(self.data_manager.redis_connection, job_id) self.write({'job': job.to_dict()}) else: raise NotImplemented def delete(self, job_id): - job = data.Job.load(self.data_manager.redis_connectino, job_id) + job = data.Job.load(self.data_manager.redis_connection, job_id) job.cancel() From 02f5979e9a98abec4321eb28417e8bd140efc95c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 18 Dec 2013 09:40:48 +0200 Subject: [PATCH 063/540] Add afork to mitigate issues of mixing threading and forking --- rd_service/cli.py | 5 +++++ rd_service/data/worker.py | 1 + rd_service/requirements.txt | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rd_service/cli.py b/rd_service/cli.py index 36648bd06..64bea6226 100644 --- a/rd_service/cli.py +++ b/rd_service/cli.py @@ -3,6 +3,11 @@ CLI to start the workers. TODO: move API server startup here. """ +import atfork +atfork.monkeypatch_os_fork_functions() +import atfork.stdlib_fixer +atfork.stdlib_fixer.fix_logging_module() + import argparse import logging import urlparse diff --git a/rd_service/data/worker.py b/rd_service/data/worker.py index c89e8236d..d75ac1951 100644 --- a/rd_service/data/worker.py +++ b/rd_service/data/worker.py @@ -237,6 +237,7 @@ class Worker(threading.Thread): annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ (pid, job.id, job.query_hash, job.priority, job.query) + # TODO: here's the part that needs to be forked, not all of the worker process... data, error = self.query_runner(annotated_query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", diff --git a/rd_service/requirements.txt b/rd_service/requirements.txt index e38d6691d..f49582580 100644 --- a/rd_service/requirements.txt +++ b/rd_service/requirements.txt @@ -6,4 +6,5 @@ Django==1.5.4 django-db-pool==0.0.10 qr==0.6.0 python-dateutil==2.1 -setproctitle==1.1.7 \ No newline at end of file +setproctitle==1.1.7 +atfork==0.1.2 \ No newline at end of file From 442da290a4c9926c1da05df14d2e08a495c37240 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 22 Dec 2013 17:05:40 +0200 Subject: [PATCH 064/540] Fix: need to make sure that each category has a value --- rd_ui/app/scripts/ng-highchart.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 2dae23aec..1e027277b 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -43,12 +43,22 @@ angular.module('highchart', []) if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { scope.chart.xAxis[0].update({type: 'category'}); + + // We need to make sure that for each category, each series has a value. + var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); + _.each(scope.series, function(s) { - _.each(s.data, function(p) { - p.name = p.x; - delete p.x; - }) - }) + // TODO: move this logic to Query#getChartData + var yValues = _.groupBy(s.data, 'x'); + + var newData = _.sortBy(_.map(categories, function(category) { + return { + name: category, + y: yValues[category][0].y + } + }), 'y'); + s.data = newData; + }); } else { scope.chart.xAxis[0].update({type: 'datetime'}); } From 42c3fcf2482832fcd3637724dbff1d60959db445 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 22 Dec 2013 17:23:39 +0200 Subject: [PATCH 065/540] having values ordered by Y, messes up the data so removing this --- rd_ui/app/scripts/ng-highchart.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 1e027277b..1131f05a6 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -46,7 +46,7 @@ angular.module('highchart', []) // We need to make sure that for each category, each series has a value. var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); - + _.each(scope.series, function(s) { // TODO: move this logic to Query#getChartData var yValues = _.groupBy(s.data, 'x'); @@ -54,9 +54,10 @@ angular.module('highchart', []) var newData = _.sortBy(_.map(categories, function(category) { return { name: category, - y: yValues[category][0].y + y: yValues[category] && yValues[category][0].y } - }), 'y'); + }), 'name'); + s.data = newData; }); } else { From 3bd8b177bed7678b3970b4e545d6056e89139151 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Dec 2013 21:23:52 +0200 Subject: [PATCH 066/540] Dashboard menu has sub-menus now --- rd_ui/app/index.html | 15 +++++++--- rd_ui/app/scripts/controllers.js | 3 +- rd_ui/app/scripts/ng-highchart.js | 2 +- rd_ui/app/styles/redash.css | 49 +++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index ce59987c5..7a311bdf2 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -38,12 +38,19 @@ diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 88bc53478..b8da0ced6 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -287,6 +287,8 @@ } return parts[0]; }); + $scope.otherDashboards = $scope.groupedDashboards['Other'] || []; + delete $scope.groupedDashboards['Other']; }); } @@ -306,7 +308,6 @@ var IndexCtrl = function($scope, Dashboard) { $scope.$parent.pageTitle = "Home"; - $scope.archiveDashboard = function(dashboard) { if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) { dashboard.$delete(function() { diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 1131f05a6..b4c445a29 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -46,7 +46,7 @@ angular.module('highchart', []) // We need to make sure that for each category, each series has a value. var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); - + _.each(scope.series, function(s) { // TODO: move this logic to Query#getChartData var yValues = _.groupBy(s.data, 'x'); diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 51257114c..aee73ddc2 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -126,4 +126,53 @@ to add those CSS styles here. */ color: #ffffff; text-decoration: none; background-color: #428bca; +} + +/* Dropdown submenus */ +.dropdown-submenu { + position: relative; +} + +.dropdown-submenu > .dropdown-menu { + top: 0; + left: 100%; + margin-top: -6px; + margin-left: -1px; + -webkit-border-radius: 0 6px 6px 6px; + -moz-border-radius: 0 6px 6px 6px; + border-radius: 0 6px 6px 6px; +} + +.dropdown-submenu:hover > .dropdown-menu { + display: block; +} + +.dropdown-submenu > a:after { + display: block; + content: " "; + float: right; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; + border-width: 5px 0 5px 5px; + border-left-color: #cccccc; + margin-top: 5px; + margin-right: -10px; +} + +.dropdown-submenu:hover > a:after { + /*border-left-color: #ffffff;*/ +} + +.dropdown-submenu.pull-left { + float: none; +} + +.dropdown-submenu.pull-left > .dropdown-menu { + left: -100%; + margin-left: 10px; + -webkit-border-radius: 6px 0 6px 6px; + -moz-border-radius: 6px 0 6px 6px; + border-radius: 6px 0 6px 6px; } \ No newline at end of file From f5f52584221b3b582753d24799e5f52659dc54d3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Dec 2013 21:32:38 +0200 Subject: [PATCH 067/540] Escape % in column names. Fixes #42 --- rd_ui/app/scripts/services.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 847e6afaa..94bb9c2d5 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -155,13 +155,15 @@ var parts = column.split('::'); var name = parts[1]; if (parts[0] != '') { - name = parts[0].replace(/ /g, '_').replace(/\?/g,''); + // TODO: it's probably time to generalize this. + // see also getColumnFriendlyName + name = parts[0].replace(/%/g, '--pct').replace(/ /g, '_').replace(/\?/g,''); } return name; } QueryResult.prototype.getColumnFriendlyName = function (column) { - return this.getColumnCleanName(column).replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { + return this.getColumnCleanName(column).replace('--pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { return a.toUpperCase(); }); } From d46c2b086c965af1eb664c3af241c76a8bbe0005 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Dec 2013 21:56:01 +0200 Subject: [PATCH 068/540] Prevent from setting a blank query title. Fixes #34 --- rd_ui/app/scripts/controllers.js | 1 - rd_ui/app/scripts/directives.js | 12 +++++++++++- rd_ui/app/views/queryfiddle.html | 4 +--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index b8da0ced6..03b5b19da 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -262,7 +262,6 @@ return $filter('refreshRateHumanize')(value); } } -// id: 672, ] $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}]; diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 313f1944f..9d739b46d 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -158,7 +158,10 @@ directives.directive('newWidgetForm', ['$http', function($http) { directives.directive('editInPlace', function () { return { restrict: 'E', - scope: { value: '=' }, + scope: { + value: '=', + ignoreBlanks: '=' + }, template: '', link: function ($scope, element, attrs) { // Let's get a reference to the input element, as we'll want to reference it. @@ -172,6 +175,10 @@ directives.directive('editInPlace', function () { // ng-click handler to activate edit-in-place $scope.edit = function () { + if ($scope.ignoreBlanks) { + $scope.oldValue = $scope.value; + } + $scope.editing = true; // We control display through a class on the directive itself. See the CSS. @@ -184,6 +191,9 @@ directives.directive('editInPlace', function () { }; $(inputElement).blur(function() { + if ($scope.ignoreBlanks && _.isEmpty($scope.value)) { + $scope.value = $scope.oldValue; + } $scope.editing = false; element.removeClass('active'); }) diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 3da0f4082..4ff57c0d8 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -3,14 +3,12 @@

    - +

    - -
    Download Data Set From ba0291dd3e9efa29a60468a0f7ea6394a8f541c0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Dec 2013 21:59:01 +0200 Subject: [PATCH 069/540] You can't have dashes in object properties. Fixes #42 --- rd_ui/app/scripts/services.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 94bb9c2d5..ab32673ce 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -157,13 +157,13 @@ if (parts[0] != '') { // TODO: it's probably time to generalize this. // see also getColumnFriendlyName - name = parts[0].replace(/%/g, '--pct').replace(/ /g, '_').replace(/\?/g,''); + name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,''); } return name; } QueryResult.prototype.getColumnFriendlyName = function (column) { - return this.getColumnCleanName(column).replace('--pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { + return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { return a.toUpperCase(); }); } From 27294fd81c170c8d0bfd24a923423e362caa5ee7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 26 Dec 2013 14:09:59 +0200 Subject: [PATCH 070/540] Limit page title size --- rd_ui/app/index.html | 2 +- rd_ui/app/styles/redash.css | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 7a311bdf2..71b1db3a2 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -33,7 +33,7 @@
      -
      • +
      • diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index aee73ddc2..aeb0268c2 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -2,6 +2,13 @@ body { padding-top: 70px; } +a.page-title { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 400px; +} + a.navbar-brand { font-style: italic; } From d4fa34fe4b45802c015e3ee802a89bb2d91033a2 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 29 Dec 2013 18:25:50 +0200 Subject: [PATCH 071/540] harcode dependencies --- rd_service/requirements.txt | 2 +- rd_ui/bower.json | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/rd_service/requirements.txt b/rd_service/requirements.txt index f49582580..dd75d2140 100644 --- a/rd_service/requirements.txt +++ b/rd_service/requirements.txt @@ -6,5 +6,5 @@ Django==1.5.4 django-db-pool==0.0.10 qr==0.6.0 python-dateutil==2.1 -setproctitle==1.1.7 +setproctitle==1.1.8 atfork==0.1.2 \ No newline at end of file diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 6779f431d..9792acc10 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -2,22 +2,22 @@ "name": "rdUi", "version": "0.1.0", "dependencies": { - "angular": "~1.0.8", - "json3": "~3.2.4", - "jquery": "~1.9.1", - "bootstrap": "~3.0.0", - "es5-shim": "~2.0.8", - "angular-moment": "~0.2.0", - "moment": "~2.1.0", - "angular-ui-bootstrap": "~0.5.0", - "angular-ui-codemirror": "~0.0.4", - "highcharts": "~3.0.1", - "underscore": "~1.5.1", + "angular": "1.0.8", + "json3": "3.2.4", + "jquery": "1.9.1", + "bootstrap": "3.0.0", + "es5-shim": "2.0.8", + "angular-moment": "0.2.0", + "moment": "2.1.0", + "angular-ui-bootstrap": "0.5.0", + "angular-ui-codemirror": "0.0.5", + "highcharts": "3.0.1", + "underscore": "1.5.1", "angular-resource": "1.0.7", - "angular-growl": "~0.3.1", + "angular-growl": "0.3.1", "pivottable": "git@github.com:arikfr/pivottable.git#master", "cornelius": "git@github.com:restorando/cornelius.git", - "gridster": "~0.2.0" + "gridster": "0.2.0" }, "devDependencies": { "angular-mocks": "~1.0.7", From 42498391ceb245e0253587efb1d0d9fbe2b9ab43 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 30 Dec 2013 14:31:23 +0200 Subject: [PATCH 072/540] Show error notification when saving a query fails (FED #1) --- rd_ui/app/scripts/app.js | 67 +++++++++++++++++++++++++------- rd_ui/app/scripts/controllers.js | 2 + rd_ui/app/styles/redash.css | 1 + 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 471cdd4e6..9835d089b 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -1,19 +1,58 @@ -angular.module('redash', ['redash.directives', 'redash.admin_controllers', 'redash.controllers', 'redash.filters', 'redash.services', - 'redash.renderers', - 'ui.codemirror', 'highchart', 'angular-growl', 'angularMoment', 'ui.bootstrap', 'smartTable.table', 'ngResource']). - config(['$routeProvider', '$locationProvider', '$compileProvider', function ($routeProvider, $locationProvider, $compileProvider) { +angular.module('redash', [ + 'redash.directives', + 'redash.admin_controllers', + 'redash.controllers', + 'redash.filters', + 'redash.services', + 'redash.renderers', + 'ui.codemirror', + 'highchart', + 'angular-growl', + 'angularMoment', + 'ui.bootstrap', + 'smartTable.table', + 'ngResource' +]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', + function($routeProvider, $locationProvider, $compileProvider, growlProvider) { $compileProvider.urlSanitizationWhitelist(/^\s*(https?|http|data):/); - $locationProvider.html5Mode(true); - $routeProvider.when('/dashboard/:dashboardSlug', {templateUrl: '/views/dashboard.html', controller: 'DashboardCtrl'}); - $routeProvider.when('/queries', {templateUrl: '/views/queries.html', controller: 'QueriesCtrl', reloadOnSearch: false}); - $routeProvider.when('/queries/new', {templateUrl: '/views/queryfiddle.html', controller: 'QueryFiddleCtrl', reloadOnSearch: false}); - $routeProvider.when('/queries/:queryId', {templateUrl: '/views/queryfiddle.html', controller: 'QueryFiddleCtrl', reloadOnSearch: false}); - $routeProvider.when('/admin/status', {templateUrl: '/views/admin_status.html', controller: 'AdminStatusCtrl'}); - $routeProvider.when('/', {templateUrl: '/views/index.html', controller: 'IndexCtrl'}); - $routeProvider.otherwise({redirectTo: '/'}); + growlProvider.globalTimeToLive(2000); - Highcharts.setOptions({colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]}); - }]); + $routeProvider.when('/dashboard/:dashboardSlug', { + templateUrl: '/views/dashboard.html', + controller: 'DashboardCtrl' + }); + $routeProvider.when('/queries', { + templateUrl: '/views/queries.html', + controller: 'QueriesCtrl', + reloadOnSearch: false + }); + $routeProvider.when('/queries/new', { + templateUrl: '/views/queryfiddle.html', + controller: 'QueryFiddleCtrl', + reloadOnSearch: false + }); + $routeProvider.when('/queries/:queryId', { + templateUrl: '/views/queryfiddle.html', + controller: 'QueryFiddleCtrl', + reloadOnSearch: false + }); + $routeProvider.when('/admin/status', { + templateUrl: '/views/admin_status.html', + controller: 'AdminStatusCtrl' + }); + $routeProvider.when('/', { + templateUrl: '/views/index.html', + controller: 'IndexCtrl' + }); + $routeProvider.otherwise({ + redirectTo: '/' + }); + Highcharts.setOptions({ + colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", + "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"] + }); + } +]); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 03b5b19da..393fe7b76 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -68,6 +68,8 @@ $location.path($location.path().replace(oldId, q.id)).replace(); } } + }, function(httpResponse) { + growl.addErrorMessage("Query could not be saved"); }); }; diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index aeb0268c2..e935e6b33 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -53,6 +53,7 @@ a.navbar-brand { right: 10px; float: right; width: 250px; + z-index: 10000; } .growl-item.ng-enter, From 6d8b256d10d2e3f930ddde6b841220134b98575a Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 30 Dec 2013 15:11:49 +0200 Subject: [PATCH 073/540] Show indication when query has unsaved changes (FED #1) --- rd_ui/app/scripts/controllers.js | 5 ++++- rd_ui/app/views/queryfiddle.html | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 393fe7b76..2a4989e8b 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -54,6 +54,8 @@ } delete $scope.query.latest_query_data; $scope.query.$save(function (q) { + $scope.pristineQuery = q.query; + if (duplicate) { growl.addInfoMessage("Query duplicated.", {ttl: 2000}); } else{ @@ -144,7 +146,8 @@ }); if ($routeParams.queryId != undefined) { - $scope.query = Query.get({id: $routeParams.queryId}, function() { + $scope.query = Query.get({id: $routeParams.queryId}, function(q) { + $scope.pristineQuery = q.query; $scope.queryResult = $scope.query.getQueryResult(); }); } else { diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 4ff57c0d8..2e436f746 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -17,7 +17,9 @@
        - +
    From 304a14e5a13fbea3c67251bc522a4d5d6b0368ff Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 30 Dec 2013 16:34:48 +0200 Subject: [PATCH 074/540] Show confirm box, when trying to leave the page before saving the query (FED #1) --- rd_ui/app/scripts/controllers.js | 30 ++++++++++++++++++++++++++---- rd_ui/app/views/queryfiddle.html | 2 +- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 2a4989e8b..aca4c7862 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -29,7 +29,23 @@ $scope.updateTime = ''; } - var QueryFiddleCtrl = function ($scope, $routeParams, $http, $location, growl, notifications, Query) { + var QueryFiddleCtrl = function ($scope, $window, $routeParams, $http, $location, growl, notifications, Query) { + var leavingPageText = "You will lose your changes if you leave"; + var pristineQuery = null; + + $window.onbeforeunload = function(){ + if ($scope.queryChanged) { + return leavingPageText; + } + } + + $scope.$on('$locationChangeStart', function(event, next, current) { + if($scope.queryChanged && + !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { + event.preventDefault(); + } + }); + $scope.$parent.pageTitle = "Query Fiddle"; $scope.tabs = [{'key': 'table', 'name': 'Table'}, {'key': 'chart', 'name': 'Chart'}, @@ -54,7 +70,8 @@ } delete $scope.query.latest_query_data; $scope.query.$save(function (q) { - $scope.pristineQuery = q.query; + pristineQuery = q.query; + $scope.queryChanged = false; if (duplicate) { growl.addInfoMessage("Query duplicated.", {ttl: 2000}); @@ -147,7 +164,7 @@ if ($routeParams.queryId != undefined) { $scope.query = Query.get({id: $routeParams.queryId}, function(q) { - $scope.pristineQuery = q.query; + pristineQuery = q.query; $scope.queryResult = $scope.query.getQueryResult(); }); } else { @@ -158,6 +175,11 @@ $scope.$watch('query.name', function() { $scope.$parent.pageTitle = $scope.query.name; }); + $scope.$watch('query.query', function(q) { + if (q) { + $scope.queryChanged = (q != pristineQuery); + } + }); $scope.executeQuery = function() { $scope.queryResult = $scope.query.getQueryResult(0); @@ -325,7 +347,7 @@ .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('QueryFiddleCtrl', ['$scope', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) + .controller('QueryFiddleCtrl', ['$scope', '$window', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 2e436f746..3614ddb1a 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -18,7 +18,7 @@
    From 0a185cf0512a5fc7684522a5fccd94a386217fa2 Mon Sep 17 00:00:00 2001 From: Timor Date: Sun, 17 Nov 2013 15:19:53 +0200 Subject: [PATCH 075/540] add vanila vagrant configuration --- vagrant/.gitignore | 1 + vagrant/Vagrantfile | 118 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 vagrant/.gitignore create mode 100644 vagrant/Vagrantfile diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 000000000..8000dd9db --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 000000000..39d18d822 --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,118 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "precise32" + + # The url from where the 'config.vm.box' box will be fetched if it + # doesn't already exist on the user's system. + config.vm.box_url = "http://files.vagrantup.com/precise32.box" + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network :forwarded_port, guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network :private_network, ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network :public_network + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider :virtualbox do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + # vb.customize ["modifyvm", :id, "--memory", "1024"] + # end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file precise32.pp in the manifests_path directory. + # + # An example Puppet manifest to provision the message of the day: + # + # # group { "puppet": + # # ensure => "present", + # # } + # # + # # File { owner => 0, group => 0, mode => 0644 } + # # + # # file { '/etc/motd': + # # content => "Welcome to your Vagrant-built virtual machine! + # # Managed by Puppet.\n" + # # } + # + # config.vm.provision :puppet do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision :chef_solo do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { :mysql_password => "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision :chef_client do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end From 5d68f46a72d8827a92cb231d1826cc0eabee6826 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Sun, 17 Nov 2013 18:16:45 +0200 Subject: [PATCH 076/540] Add .DS_Store to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a6086e0d1..3dd65166f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc rd_service/settings.py rd_ui/dist +.DS_Store From 82fc2eb812a144b639735b1e38d9335824ab2141 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Wed, 20 Nov 2013 14:22:59 +0200 Subject: [PATCH 077/540] Still hitting the encoding conversion error --- cookbooks/redash/Berksfile | 1 + cookbooks/redash/README.md | 73 ++++++++++ cookbooks/redash/attributes/default.rb | 39 ++++++ cookbooks/redash/metadata.rb | 17 +++ cookbooks/redash/recipes/default.rb | 125 +++++++++++++++++ .../redash/templates/default/settings.py.erb | 38 ++++++ .../default/sv-redash-server-log-run.erb | 3 + .../default/sv-redash-server-run.erb | 4 + .../default/sv-redash-worker-log-run.erb | 3 + .../default/sv-redash-worker-run.erb | 4 + vagrant/Berksfile | 2 + vagrant/Berksfile.lock | 67 +++++++++ vagrant/Vagrantfile | 129 +++--------------- vagrant/notes | 5 + 14 files changed, 403 insertions(+), 107 deletions(-) create mode 100644 cookbooks/redash/Berksfile create mode 100644 cookbooks/redash/README.md create mode 100644 cookbooks/redash/attributes/default.rb create mode 100644 cookbooks/redash/metadata.rb create mode 100644 cookbooks/redash/recipes/default.rb create mode 100644 cookbooks/redash/templates/default/settings.py.erb create mode 100644 cookbooks/redash/templates/default/sv-redash-server-log-run.erb create mode 100644 cookbooks/redash/templates/default/sv-redash-server-run.erb create mode 100644 cookbooks/redash/templates/default/sv-redash-worker-log-run.erb create mode 100644 cookbooks/redash/templates/default/sv-redash-worker-run.erb create mode 100644 vagrant/Berksfile create mode 100644 vagrant/Berksfile.lock create mode 100644 vagrant/notes diff --git a/cookbooks/redash/Berksfile b/cookbooks/redash/Berksfile new file mode 100644 index 000000000..850a120dc --- /dev/null +++ b/cookbooks/redash/Berksfile @@ -0,0 +1 @@ +metadata diff --git a/cookbooks/redash/README.md b/cookbooks/redash/README.md new file mode 100644 index 000000000..b112cdb6c --- /dev/null +++ b/cookbooks/redash/README.md @@ -0,0 +1,73 @@ +redash Cookbook +================= +TODO: Enter the cookbook description here. + +e.g. +This cookbook makes your favorite breakfast sandwich. + +Requirements +------------ +TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. + +Expect to have these attriburtes defined: +node['postgresql']['password']['postgres'] <=== used for the initial connection to PG, where the redash role+databases are created + + + +e.g. +#### packages +- `toaster` - thelight needs toaster to brown your bagel. + +Attributes +---------- +TODO: List you cookbook attributes here. + +e.g. +#### thelight::default + + + + + + + + + + + + + +
    KeyTypeDescriptionDefault
    ['thelight']['bacon']Booleanwhether to include bacontrue
    + +Usage +----- +#### thelight::default +TODO: Write usage instructions for each cookbook. + +e.g. +Just include `thelight` in your node's `run_list`: + +```json +{ + "name":"my_node", + "run_list": [ + "recipe[thelight]" + ] +} +``` + +Contributing +------------ +TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. + +e.g. +1. Fork the repository on Github +2. Create a named feature branch (like `add_component_x`) +3. Write your change +4. Write tests for your change (if applicable) +5. Run the tests, ensuring they all pass +6. Submit a Pull Request using Github + +License and Authors +------------------- +Authors: TODO: List authors diff --git a/cookbooks/redash/attributes/default.rb b/cookbooks/redash/attributes/default.rb new file mode 100644 index 000000000..94628bfc8 --- /dev/null +++ b/cookbooks/redash/attributes/default.rb @@ -0,0 +1,39 @@ + +default["redash"]["install_path"] = "/opt" + +default["redash"]["user"] = "redash" + +default['redash']['redis_url'] = "redis://localhost:6379" + +default['redash']['db']['user'] = "redash" +default['redash']['db']['password'] = "secureredisecure" +#default['redash']['db']['host'] = "localhost" +default['redash']['db']['port'] = 5432 +default['redash']['db']['dbname'] = "redash_db0" +default['redash']['cfg']['dbname'] = "redash_cfgdb0" +default['redash']['cfg']['user'] = "redash" +default['redash']['cfg']['password'] = "secureredisecure" +default['redash']['cfg']['host'] = node['redash']['db']['host'] +default['redash']['cfg']['port'] = 5432 +default['redash']['db']['postgres_pwd'] = node['postgresql']['password']['postgres'] +default['redash']['cfg']['postgres_pwd'] = node['postgresql']['password']['postgres'] + + +#Accept logins from open id's verified by google accounts: +default['redash']['allow']['google_app_domain'] = "everything.me" +#Two strings with python list inside...: +default['redash']['allow']['google_app_users'] = ['joe@gmail.com','max@gmail.com'] +default['redash']['allow']['admins'] = ['timor@everything.me','arik@everything.me'] + +default['redash']['workers_count'] = 2 +default['redash']['max_connections'] = 3 + +default['redash']['cookie_secret'] = "c292a0a3aa32397cdb050e233733900f" + +default['redash']['server']['log'] = node['redash']['install_path']+"/redash/log/server.log" +default['redash']['worker']['log'] = node['redash']['install_path']+"/redash/log/worker.log" + +default['redash']['server']['py'] = "./main" +default['redash']['worker']['py'] = "./main" + +default['redash']['svlog_opt'] = "-tt" \ No newline at end of file diff --git a/cookbooks/redash/metadata.rb b/cookbooks/redash/metadata.rb new file mode 100644 index 000000000..caa9fcb90 --- /dev/null +++ b/cookbooks/redash/metadata.rb @@ -0,0 +1,17 @@ +name 'redash' +maintainer 'Everything.me' +maintainer_email 'timor@everything.me' +license 'All rights reserved' +description 'Installs/Configures redash' +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version '0.4' + +depends "postgresql" +depends "python" +depends "ark" +depends "database" + +%w(python dhcp sysctl git).each do |cookbook| + depends cookbook +end +depends 'runit', '>= 1.1.0' diff --git a/cookbooks/redash/recipes/default.rb b/cookbooks/redash/recipes/default.rb new file mode 100644 index 000000000..dda97ca5a --- /dev/null +++ b/cookbooks/redash/recipes/default.rb @@ -0,0 +1,125 @@ +# +# Cookbook Name:: redash +# Recipe:: default +# + +include_recipe "postgresql::client" +include_recipe "python" +include_recipe "runit" + +#Enable cheff to interact with pg: +include_recipe "database::postgresql" + +#Download and deploy the redash release +#TODO: version should be acc. to what's in metadata.rb + +#TODO: install path should be a configurable attribute +user node['redash']['user'] do + system true +end + + +#Ark fails due to errors in remote_file not telling rest-client to expect binary.. +#solution: use an older rest-client.. +chef_gem "rest-client__p" do + action :purge + package_name "rest-client" +end +chef_gem "rest-client__i" do + action :install + package_name "rest-client" + version "1.5.0" +end + +ark "bla" do + #url "http://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz" + #url "http://www.xmlcan.ca/~timor/redash.35.tar.gz" + url "http://www.xmlcan.ca/~timor/bla.tar.gz" + action :put + path node["redash"]["install_path"] + + #Due to peculiarity of the way the archive gets created: + strip_leading_dir false +end + +#Install dependencies acc. to file: +bash ":install pip dependencies" do + code <<-EOS + cd #{node["redash"]["install_path"]}/redash + pip install -r ./rd_service/requirements.txt + EOS +end + +#Configure: +template "#{node["redash"]["install_path"]}/redash/rd_service/settings.py" do + source "settings.py.erb" +end + +#Setup pg user(s) and database(s): +pg_db_super_connection = { + :host => node['redash']['db']['host'], + :port => node['redash']['db']['port'], + :username => 'postgres', + :password => node['redash']['db']['postgres_pwd'] +} +pg_cfg_super_connection = { + :host => node['redash']['cfg']['host'], + :port => node['redash']['cfg']['port'], + :username => 'postgres', + :password => node['redash']['cfg']['postgres_pwd'] +} + +#The data db: +postgresql_database node['redash']['db']['dbname'] do + connection pg_db_super_connection + action :create +end + +#The configuration db: +postgresql_database node['redash']['cfg']['dbname'] do + connection pg_cfg_super_connection + action :create +end + + +postgresql_database_user "redash_db_user" do + username node['redash']['db']['user'] + connection pg_db_super_connection + password node['redash']['db']['password'] + database_name node['redash']['db']['dbname'] + privileges [:all] + action [:create,:grant] +end + +postgresql_database_user "redash_cfg_user" do + username node['redash']['cfg']['user'] + connection pg_cfg_super_connection + password node['redash']['cfg']['password'] + database_name node['redash']['cfg']['dbname'] + privileges [:all] + action [:create,:grant] +end + + +#Initialize the DB, connecting as normal user: +#Setup pg user(s) and database(s): +pg_db_connection = { + :host => node['redash']['db']['host'], + :port => node['redash']['db']['port'], + :user => node['redash']['db']['user'], + :password => node['redash']['db']['password'], + :dbname => node['redash']['cfg']['dbname'] +} +constr = pg_db_connection.map{|(k,v)| "#{k}=#{v}"}.join(" ") +bash ":initialize db" do + code <<-EOS + cd #{node["redash"]["install_path"]}/redash + + psql "#{constr}" < ./rd_service/data/tables.sql + EOS +end + + +#Install runit scripts and bring the system up: +runit_service "redash-server" +runit_service "redash-worker" diff --git a/cookbooks/redash/templates/default/settings.py.erb b/cookbooks/redash/templates/default/settings.py.erb new file mode 100644 index 000000000..386af10ad --- /dev/null +++ b/cookbooks/redash/templates/default/settings.py.erb @@ -0,0 +1,38 @@ +""" +Settings module generated and dropped off by Chef. +Will be overwritten. +""" + +import django.conf + +REDIS_URL = "<%= node['redash']['redis_url'] %>" +# Connection string for the database that is used to run queries against +CONNECTION_STRING = "user=<%= node['redash']['db']['user'] %> password=<%= node['redash']['db']['password'] %> host=<%= node['redash']['db']['host'] %> port=<%= node['redash']['db']['port'] %> dbname=<%= node['redash']['db']['dbname'] %>" +# Connection string for the operational databases (where we store the queries, results, etc) +INTERNAL_DB_CONNECTION_STRING = "user=<%= node['redash']['cfg']['user'] %> password=<%= node['redash']['cfg']['password'] %> host=<%= node['redash']['cfg']['host'] %> port=<%= node['redash']['cfg']['port'] %> dbname=<%= node['redash']['cfg']['dbname'] %>" +# Google Apps domain to allow access from; any user with email in this Google Apps will be allowed +# access +GOOGLE_APPS_DOMAIN = "<%= node['redash']['allow']['google_app_domain'] %>" +# Email addresses of specific users not from the above set Google Apps Domain, that you want to +# allow access to re:dash +ALLOWED_USERS = <%= Chef::JSONCompat.to_json node['redash']['allow']['google_app_users'] %> +# Email addresses of admin users +ADMINS = <%= Chef::JSONCompat.to_json node['redash']['allow']['admins'] %> + +STATIC_ASSETS_PATH = "<%= node['redash']['install_path'] %>/rd_ui/dist/" + +WORKERS_COUNT = <%= node['redash']['workers_count'] %> +MAX_CONNECTIONS = <%= node['redash']['max_connections'] %> + +COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" + +# Configuration of the operational database for the Django models +django.conf.settings.configure(DATABASES = { 'default': { + 'ENGINE': 'dbpool.db.backends.postgresql_psycopg2', + 'OPTIONS': {'MAX_CONNS': <%= node['redash']['max_connections'] %>, 'MIN_CONNS': 1}, + 'NAME': '<%= node['redash']['db']['dbname'] %>', + 'USER': '<%= node['redash']['db']['user'] %>', + 'PASSWORD': '<%= node['redash']['db']['password'] %>', + 'HOST': '<%= node['redash']['db']['host'] %>', + 'PORT': '<%= node['redash']['db']['port'] %>', + },}, TIME_ZONE = 'UTC') diff --git a/cookbooks/redash/templates/default/sv-redash-server-log-run.erb b/cookbooks/redash/templates/default/sv-redash-server-log-run.erb new file mode 100644 index 000000000..981434db3 --- /dev/null +++ b/cookbooks/redash/templates/default/sv-redash-server-log-run.erb @@ -0,0 +1,3 @@ +#!/bin/sh +exec svlogd <%= node['redash']['svlog_opt'] %> <%= node['redash']['server']['log'] %> + diff --git a/cookbooks/redash/templates/default/sv-redash-server-run.erb b/cookbooks/redash/templates/default/sv-redash-server-run.erb new file mode 100644 index 000000000..6a4b1f1ee --- /dev/null +++ b/cookbooks/redash/templates/default/sv-redash-server-run.erb @@ -0,0 +1,4 @@ +#!/bin/sh +exec 2>&1 +exec chpst -u <%= node['redash']['user'] %> env python <%= node['redash']['server']['py'] %> + diff --git a/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb b/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb new file mode 100644 index 000000000..bea350159 --- /dev/null +++ b/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb @@ -0,0 +1,3 @@ +#!/bin/sh +exec svlogd <%= node['redash']['svlog_opt'] %> <%= node['redash']['worker']['log'] %> + diff --git a/cookbooks/redash/templates/default/sv-redash-worker-run.erb b/cookbooks/redash/templates/default/sv-redash-worker-run.erb new file mode 100644 index 000000000..60563e544 --- /dev/null +++ b/cookbooks/redash/templates/default/sv-redash-worker-run.erb @@ -0,0 +1,4 @@ +#!/bin/sh +exec 2>&1 +exec chpst -u <%= node['redash']['user'] %> env python <%= node['redash']['worker']['py'] %> + diff --git a/vagrant/Berksfile b/vagrant/Berksfile new file mode 100644 index 000000000..773ffdab0 --- /dev/null +++ b/vagrant/Berksfile @@ -0,0 +1,2 @@ +cookbook 'redash', path: '../cookbooks/redash' +cookbook 'apt' diff --git a/vagrant/Berksfile.lock b/vagrant/Berksfile.lock new file mode 100644 index 000000000..a84f20756 --- /dev/null +++ b/vagrant/Berksfile.lock @@ -0,0 +1,67 @@ +{ + "sources": { + "redash": { + "path": "../cookbooks/redash" + }, + "apt": { + "locked_version": "2.3.0" + }, + "postgresql": { + "locked_version": "3.2.0" + }, + "build-essential": { + "locked_version": "1.4.2" + }, + "openssl": { + "locked_version": "1.1.0" + }, + "python": { + "locked_version": "1.4.4" + }, + "yum": { + "locked_version": "2.4.2" + }, + "ark": { + "locked_version": "0.4.0" + }, + "database": { + "locked_version": "1.5.2" + }, + "mysql": { + "locked_version": "4.0.4" + }, + "aws": { + "locked_version": "1.0.0" + }, + "xfs": { + "locked_version": "1.1.0" + }, + "dhcp": { + "locked_version": "2.0.1" + }, + "ruby-helper": { + "locked_version": "0.0.1" + }, + "helpers-databags": { + "locked_version": "1.0.0" + }, + "sysctl": { + "locked_version": "0.3.4" + }, + "git": { + "locked_version": "2.7.0" + }, + "dmg": { + "locked_version": "2.0.8" + }, + "windows": { + "locked_version": "1.11.0" + }, + "chef_handler": { + "locked_version": "1.1.4" + }, + "runit": { + "locked_version": "1.4.0" + } + } +} diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 39d18d822..f60de9eed 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -5,114 +5,29 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "precise32" - # The url from where the 'config.vm.box' box will be fetched if it - # doesn't already exist on the user's system. - config.vm.box_url = "http://files.vagrantup.com/precise32.box" + config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box" - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # config.vm.network :forwarded_port, guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network :private_network, ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network :public_network - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - # config.ssh.forward_agent = true - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - # config.vm.provider :virtualbox do |vb| - # # Don't boot with headless mode - # vb.gui = true - # - # # Use VBoxManage to customize the VM. For example to change memory: - # vb.customize ["modifyvm", :id, "--memory", "1024"] - # end - # - # View the documentation for the provider you're using for more - # information on available options. - - # Enable provisioning with Puppet stand alone. Puppet manifests - # are contained in a directory path relative to this Vagrantfile. - # You will need to create the manifests directory and a manifest in - # the file precise32.pp in the manifests_path directory. - # - # An example Puppet manifest to provision the message of the day: - # - # # group { "puppet": - # # ensure => "present", - # # } - # # - # # File { owner => 0, group => 0, mode => 0644 } - # # - # # file { '/etc/motd': - # # content => "Welcome to your Vagrant-built virtual machine! - # # Managed by Puppet.\n" - # # } - # - # config.vm.provision :puppet do |puppet| - # puppet.manifests_path = "manifests" - # puppet.manifest_file = "site.pp" - # end - - # Enable provisioning with chef solo, specifying a cookbooks path, roles - # path, and data_bags path (all relative to this Vagrantfile), and adding - # some recipes and/or roles. - # - # config.vm.provision :chef_solo do |chef| - # chef.cookbooks_path = "../my-recipes/cookbooks" - # chef.roles_path = "../my-recipes/roles" - # chef.data_bags_path = "../my-recipes/data_bags" - # chef.add_recipe "mysql" - # chef.add_role "web" - # - # # You may also specify custom JSON attributes: - # chef.json = { :mysql_password => "foo" } - # end - - # Enable provisioning with chef server, specifying the chef server URL, - # and the path to the validation key (relative to this Vagrantfile). - # - # The Opscode Platform uses HTTPS. Substitute your organization for - # ORGNAME in the URL and validation key. - # - # If you have your own Chef Server, use the appropriate URL, which may be - # HTTP instead of HTTPS depending on your configuration. Also change the - # validation key to validation.pem. - # - # config.vm.provision :chef_client do |chef| - # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" - # chef.validation_key_path = "ORGNAME-validator.pem" - # end - # - # If you're using the Opscode platform, your validator client is - # ORGNAME-validator, replacing ORGNAME with your organization name. - # - # If you have your own Chef Server, the default validation client name is - # chef-validator, unless you changed the configuration. - # - # chef.validation_client_name = "ORGNAME-validator" + config.berkshelf.enabled = true + #config.omnibus.chef_version = :latest + config.omnibus.chef_version = "11.6.0" + + config.vm.network "forwarded_port", guest: 8888, host: 9999 + + + config.vm.provision :chef_solo do |chef| + #Run apt-get update before anything else (specifically postgresql).. + chef.add_recipe "apt" + chef.add_recipe "redash" + chef.add_recipe "postgresql" + + chef.json = { + "apt" => {"compiletime" => true}, + "postgresql" => {"password" => {"postgres" => "securepass"}}, + "redash" => {"db" => {"host" => "localhost"}}, + } + + end end diff --git a/vagrant/notes b/vagrant/notes new file mode 100644 index 000000000..2895a84ff --- /dev/null +++ b/vagrant/notes @@ -0,0 +1,5 @@ +Download: +https://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz +--> /opt/redash +deploy postgresql db, sql file. +edit / replace settings file. From 74142187c155b6339c92b1d9856bcff20ac677c9 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Wed, 20 Nov 2013 16:08:08 +0200 Subject: [PATCH 078/540] Cleanup --- cookbooks/redash/recipes/default.rb | 22 +++------- vagrant/.gitignore | 1 + vagrant/Berksfile.lock | 67 ----------------------------- vagrant/notes | 5 --- 4 files changed, 6 insertions(+), 89 deletions(-) delete mode 100644 vagrant/Berksfile.lock delete mode 100644 vagrant/notes diff --git a/cookbooks/redash/recipes/default.rb b/cookbooks/redash/recipes/default.rb index dda97ca5a..57e4a119d 100644 --- a/cookbooks/redash/recipes/default.rb +++ b/cookbooks/redash/recipes/default.rb @@ -3,6 +3,9 @@ # Recipe:: default # +#Workaround for endocind errors with remote_file: +Encoding.default_external = Encoding::ASCII_8BIT + include_recipe "postgresql::client" include_recipe "python" include_recipe "runit" @@ -18,23 +21,8 @@ user node['redash']['user'] do system true end - -#Ark fails due to errors in remote_file not telling rest-client to expect binary.. -#solution: use an older rest-client.. -chef_gem "rest-client__p" do - action :purge - package_name "rest-client" -end -chef_gem "rest-client__i" do - action :install - package_name "rest-client" - version "1.5.0" -end - -ark "bla" do - #url "http://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz" - #url "http://www.xmlcan.ca/~timor/redash.35.tar.gz" - url "http://www.xmlcan.ca/~timor/bla.tar.gz" +ark "redash" do + url "http://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz" action :put path node["redash"]["install_path"] diff --git a/vagrant/.gitignore b/vagrant/.gitignore index 8000dd9db..43c013aee 100644 --- a/vagrant/.gitignore +++ b/vagrant/.gitignore @@ -1 +1,2 @@ .vagrant +Berksfile.lock diff --git a/vagrant/Berksfile.lock b/vagrant/Berksfile.lock deleted file mode 100644 index a84f20756..000000000 --- a/vagrant/Berksfile.lock +++ /dev/null @@ -1,67 +0,0 @@ -{ - "sources": { - "redash": { - "path": "../cookbooks/redash" - }, - "apt": { - "locked_version": "2.3.0" - }, - "postgresql": { - "locked_version": "3.2.0" - }, - "build-essential": { - "locked_version": "1.4.2" - }, - "openssl": { - "locked_version": "1.1.0" - }, - "python": { - "locked_version": "1.4.4" - }, - "yum": { - "locked_version": "2.4.2" - }, - "ark": { - "locked_version": "0.4.0" - }, - "database": { - "locked_version": "1.5.2" - }, - "mysql": { - "locked_version": "4.0.4" - }, - "aws": { - "locked_version": "1.0.0" - }, - "xfs": { - "locked_version": "1.1.0" - }, - "dhcp": { - "locked_version": "2.0.1" - }, - "ruby-helper": { - "locked_version": "0.0.1" - }, - "helpers-databags": { - "locked_version": "1.0.0" - }, - "sysctl": { - "locked_version": "0.3.4" - }, - "git": { - "locked_version": "2.7.0" - }, - "dmg": { - "locked_version": "2.0.8" - }, - "windows": { - "locked_version": "1.11.0" - }, - "chef_handler": { - "locked_version": "1.1.4" - }, - "runit": { - "locked_version": "1.4.0" - } - } -} diff --git a/vagrant/notes b/vagrant/notes deleted file mode 100644 index 2895a84ff..000000000 --- a/vagrant/notes +++ /dev/null @@ -1,5 +0,0 @@ -Download: -https://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz ---> /opt/redash -deploy postgresql db, sql file. -edit / replace settings file. From a3cf6711a55bc80e7c5d2299e507409155bd18d1 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Wed, 20 Nov 2013 16:54:40 +0200 Subject: [PATCH 079/540] Remove chef cookbooks (migrated to separate repo) --- cookbooks/redash/Berksfile | 1 - cookbooks/redash/README.md | 73 ----------- cookbooks/redash/attributes/default.rb | 39 ------ cookbooks/redash/metadata.rb | 17 --- cookbooks/redash/recipes/default.rb | 113 ------------------ .../redash/templates/default/settings.py.erb | 38 ------ .../default/sv-redash-server-log-run.erb | 3 - .../default/sv-redash-server-run.erb | 4 - .../default/sv-redash-worker-log-run.erb | 3 - .../default/sv-redash-worker-run.erb | 4 - 10 files changed, 295 deletions(-) delete mode 100644 cookbooks/redash/Berksfile delete mode 100644 cookbooks/redash/README.md delete mode 100644 cookbooks/redash/attributes/default.rb delete mode 100644 cookbooks/redash/metadata.rb delete mode 100644 cookbooks/redash/recipes/default.rb delete mode 100644 cookbooks/redash/templates/default/settings.py.erb delete mode 100644 cookbooks/redash/templates/default/sv-redash-server-log-run.erb delete mode 100644 cookbooks/redash/templates/default/sv-redash-server-run.erb delete mode 100644 cookbooks/redash/templates/default/sv-redash-worker-log-run.erb delete mode 100644 cookbooks/redash/templates/default/sv-redash-worker-run.erb diff --git a/cookbooks/redash/Berksfile b/cookbooks/redash/Berksfile deleted file mode 100644 index 850a120dc..000000000 --- a/cookbooks/redash/Berksfile +++ /dev/null @@ -1 +0,0 @@ -metadata diff --git a/cookbooks/redash/README.md b/cookbooks/redash/README.md deleted file mode 100644 index b112cdb6c..000000000 --- a/cookbooks/redash/README.md +++ /dev/null @@ -1,73 +0,0 @@ -redash Cookbook -================= -TODO: Enter the cookbook description here. - -e.g. -This cookbook makes your favorite breakfast sandwich. - -Requirements ------------- -TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. - -Expect to have these attriburtes defined: -node['postgresql']['password']['postgres'] <=== used for the initial connection to PG, where the redash role+databases are created - - - -e.g. -#### packages -- `toaster` - thelight needs toaster to brown your bagel. - -Attributes ----------- -TODO: List you cookbook attributes here. - -e.g. -#### thelight::default - - - - - - - - - - - - - -
    KeyTypeDescriptionDefault
    ['thelight']['bacon']Booleanwhether to include bacontrue
    - -Usage ------ -#### thelight::default -TODO: Write usage instructions for each cookbook. - -e.g. -Just include `thelight` in your node's `run_list`: - -```json -{ - "name":"my_node", - "run_list": [ - "recipe[thelight]" - ] -} -``` - -Contributing ------------- -TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. - -e.g. -1. Fork the repository on Github -2. Create a named feature branch (like `add_component_x`) -3. Write your change -4. Write tests for your change (if applicable) -5. Run the tests, ensuring they all pass -6. Submit a Pull Request using Github - -License and Authors -------------------- -Authors: TODO: List authors diff --git a/cookbooks/redash/attributes/default.rb b/cookbooks/redash/attributes/default.rb deleted file mode 100644 index 94628bfc8..000000000 --- a/cookbooks/redash/attributes/default.rb +++ /dev/null @@ -1,39 +0,0 @@ - -default["redash"]["install_path"] = "/opt" - -default["redash"]["user"] = "redash" - -default['redash']['redis_url'] = "redis://localhost:6379" - -default['redash']['db']['user'] = "redash" -default['redash']['db']['password'] = "secureredisecure" -#default['redash']['db']['host'] = "localhost" -default['redash']['db']['port'] = 5432 -default['redash']['db']['dbname'] = "redash_db0" -default['redash']['cfg']['dbname'] = "redash_cfgdb0" -default['redash']['cfg']['user'] = "redash" -default['redash']['cfg']['password'] = "secureredisecure" -default['redash']['cfg']['host'] = node['redash']['db']['host'] -default['redash']['cfg']['port'] = 5432 -default['redash']['db']['postgres_pwd'] = node['postgresql']['password']['postgres'] -default['redash']['cfg']['postgres_pwd'] = node['postgresql']['password']['postgres'] - - -#Accept logins from open id's verified by google accounts: -default['redash']['allow']['google_app_domain'] = "everything.me" -#Two strings with python list inside...: -default['redash']['allow']['google_app_users'] = ['joe@gmail.com','max@gmail.com'] -default['redash']['allow']['admins'] = ['timor@everything.me','arik@everything.me'] - -default['redash']['workers_count'] = 2 -default['redash']['max_connections'] = 3 - -default['redash']['cookie_secret'] = "c292a0a3aa32397cdb050e233733900f" - -default['redash']['server']['log'] = node['redash']['install_path']+"/redash/log/server.log" -default['redash']['worker']['log'] = node['redash']['install_path']+"/redash/log/worker.log" - -default['redash']['server']['py'] = "./main" -default['redash']['worker']['py'] = "./main" - -default['redash']['svlog_opt'] = "-tt" \ No newline at end of file diff --git a/cookbooks/redash/metadata.rb b/cookbooks/redash/metadata.rb deleted file mode 100644 index caa9fcb90..000000000 --- a/cookbooks/redash/metadata.rb +++ /dev/null @@ -1,17 +0,0 @@ -name 'redash' -maintainer 'Everything.me' -maintainer_email 'timor@everything.me' -license 'All rights reserved' -description 'Installs/Configures redash' -long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.4' - -depends "postgresql" -depends "python" -depends "ark" -depends "database" - -%w(python dhcp sysctl git).each do |cookbook| - depends cookbook -end -depends 'runit', '>= 1.1.0' diff --git a/cookbooks/redash/recipes/default.rb b/cookbooks/redash/recipes/default.rb deleted file mode 100644 index 57e4a119d..000000000 --- a/cookbooks/redash/recipes/default.rb +++ /dev/null @@ -1,113 +0,0 @@ -# -# Cookbook Name:: redash -# Recipe:: default -# - -#Workaround for endocind errors with remote_file: -Encoding.default_external = Encoding::ASCII_8BIT - -include_recipe "postgresql::client" -include_recipe "python" -include_recipe "runit" - -#Enable cheff to interact with pg: -include_recipe "database::postgresql" - -#Download and deploy the redash release -#TODO: version should be acc. to what's in metadata.rb - -#TODO: install path should be a configurable attribute -user node['redash']['user'] do - system true -end - -ark "redash" do - url "http://github.com/EverythingMe/redash/releases/download/v0.1.35/redash.35.tar.gz" - action :put - path node["redash"]["install_path"] - - #Due to peculiarity of the way the archive gets created: - strip_leading_dir false -end - -#Install dependencies acc. to file: -bash ":install pip dependencies" do - code <<-EOS - cd #{node["redash"]["install_path"]}/redash - pip install -r ./rd_service/requirements.txt - EOS -end - -#Configure: -template "#{node["redash"]["install_path"]}/redash/rd_service/settings.py" do - source "settings.py.erb" -end - -#Setup pg user(s) and database(s): -pg_db_super_connection = { - :host => node['redash']['db']['host'], - :port => node['redash']['db']['port'], - :username => 'postgres', - :password => node['redash']['db']['postgres_pwd'] -} -pg_cfg_super_connection = { - :host => node['redash']['cfg']['host'], - :port => node['redash']['cfg']['port'], - :username => 'postgres', - :password => node['redash']['cfg']['postgres_pwd'] -} - -#The data db: -postgresql_database node['redash']['db']['dbname'] do - connection pg_db_super_connection - action :create -end - -#The configuration db: -postgresql_database node['redash']['cfg']['dbname'] do - connection pg_cfg_super_connection - action :create -end - - -postgresql_database_user "redash_db_user" do - username node['redash']['db']['user'] - connection pg_db_super_connection - password node['redash']['db']['password'] - database_name node['redash']['db']['dbname'] - privileges [:all] - action [:create,:grant] -end - -postgresql_database_user "redash_cfg_user" do - username node['redash']['cfg']['user'] - connection pg_cfg_super_connection - password node['redash']['cfg']['password'] - database_name node['redash']['cfg']['dbname'] - privileges [:all] - action [:create,:grant] -end - - -#Initialize the DB, connecting as normal user: -#Setup pg user(s) and database(s): -pg_db_connection = { - :host => node['redash']['db']['host'], - :port => node['redash']['db']['port'], - :user => node['redash']['db']['user'], - :password => node['redash']['db']['password'], - :dbname => node['redash']['cfg']['dbname'] -} -constr = pg_db_connection.map{|(k,v)| "#{k}=#{v}"}.join(" ") -bash ":initialize db" do - code <<-EOS - cd #{node["redash"]["install_path"]}/redash - - psql "#{constr}" < ./rd_service/data/tables.sql - EOS -end - - -#Install runit scripts and bring the system up: -runit_service "redash-server" -runit_service "redash-worker" diff --git a/cookbooks/redash/templates/default/settings.py.erb b/cookbooks/redash/templates/default/settings.py.erb deleted file mode 100644 index 386af10ad..000000000 --- a/cookbooks/redash/templates/default/settings.py.erb +++ /dev/null @@ -1,38 +0,0 @@ -""" -Settings module generated and dropped off by Chef. -Will be overwritten. -""" - -import django.conf - -REDIS_URL = "<%= node['redash']['redis_url'] %>" -# Connection string for the database that is used to run queries against -CONNECTION_STRING = "user=<%= node['redash']['db']['user'] %> password=<%= node['redash']['db']['password'] %> host=<%= node['redash']['db']['host'] %> port=<%= node['redash']['db']['port'] %> dbname=<%= node['redash']['db']['dbname'] %>" -# Connection string for the operational databases (where we store the queries, results, etc) -INTERNAL_DB_CONNECTION_STRING = "user=<%= node['redash']['cfg']['user'] %> password=<%= node['redash']['cfg']['password'] %> host=<%= node['redash']['cfg']['host'] %> port=<%= node['redash']['cfg']['port'] %> dbname=<%= node['redash']['cfg']['dbname'] %>" -# Google Apps domain to allow access from; any user with email in this Google Apps will be allowed -# access -GOOGLE_APPS_DOMAIN = "<%= node['redash']['allow']['google_app_domain'] %>" -# Email addresses of specific users not from the above set Google Apps Domain, that you want to -# allow access to re:dash -ALLOWED_USERS = <%= Chef::JSONCompat.to_json node['redash']['allow']['google_app_users'] %> -# Email addresses of admin users -ADMINS = <%= Chef::JSONCompat.to_json node['redash']['allow']['admins'] %> - -STATIC_ASSETS_PATH = "<%= node['redash']['install_path'] %>/rd_ui/dist/" - -WORKERS_COUNT = <%= node['redash']['workers_count'] %> -MAX_CONNECTIONS = <%= node['redash']['max_connections'] %> - -COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" - -# Configuration of the operational database for the Django models -django.conf.settings.configure(DATABASES = { 'default': { - 'ENGINE': 'dbpool.db.backends.postgresql_psycopg2', - 'OPTIONS': {'MAX_CONNS': <%= node['redash']['max_connections'] %>, 'MIN_CONNS': 1}, - 'NAME': '<%= node['redash']['db']['dbname'] %>', - 'USER': '<%= node['redash']['db']['user'] %>', - 'PASSWORD': '<%= node['redash']['db']['password'] %>', - 'HOST': '<%= node['redash']['db']['host'] %>', - 'PORT': '<%= node['redash']['db']['port'] %>', - },}, TIME_ZONE = 'UTC') diff --git a/cookbooks/redash/templates/default/sv-redash-server-log-run.erb b/cookbooks/redash/templates/default/sv-redash-server-log-run.erb deleted file mode 100644 index 981434db3..000000000 --- a/cookbooks/redash/templates/default/sv-redash-server-log-run.erb +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -exec svlogd <%= node['redash']['svlog_opt'] %> <%= node['redash']['server']['log'] %> - diff --git a/cookbooks/redash/templates/default/sv-redash-server-run.erb b/cookbooks/redash/templates/default/sv-redash-server-run.erb deleted file mode 100644 index 6a4b1f1ee..000000000 --- a/cookbooks/redash/templates/default/sv-redash-server-run.erb +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -exec 2>&1 -exec chpst -u <%= node['redash']['user'] %> env python <%= node['redash']['server']['py'] %> - diff --git a/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb b/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb deleted file mode 100644 index bea350159..000000000 --- a/cookbooks/redash/templates/default/sv-redash-worker-log-run.erb +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -exec svlogd <%= node['redash']['svlog_opt'] %> <%= node['redash']['worker']['log'] %> - diff --git a/cookbooks/redash/templates/default/sv-redash-worker-run.erb b/cookbooks/redash/templates/default/sv-redash-worker-run.erb deleted file mode 100644 index 60563e544..000000000 --- a/cookbooks/redash/templates/default/sv-redash-worker-run.erb +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -exec 2>&1 -exec chpst -u <%= node['redash']['user'] %> env python <%= node['redash']['worker']['py'] %> - From 5b286b74b0d3378a643eeb4bfae85d68af034fc5 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Thu, 21 Nov 2013 19:54:47 +0200 Subject: [PATCH 080/540] Code review by Yoni --- vagrant/Berksfile | 4 +++- vagrant/Vagrantfile | 33 ++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/vagrant/Berksfile b/vagrant/Berksfile index 773ffdab0..91560b352 100644 --- a/vagrant/Berksfile +++ b/vagrant/Berksfile @@ -1,2 +1,4 @@ -cookbook 'redash', path: '../cookbooks/redash' cookbook 'apt' +cookbook 'postgresql' +#cookbook 'redash', git: 'git@gitlab.doit9.com:ops/chef-redash.git' +cookbook 'redash', path: '../../chef-redash' diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index f60de9eed..8e762be6f 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -4,30 +4,33 @@ # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = "2" -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.box = "precise32" +# TODO: should be result of encryption of readable password +__pg_password = "securepass" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "ubuntu-precise-cloudimg-amd64" config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box" - config.berkshelf.enabled = true - #config.omnibus.chef_version = :latest - config.omnibus.chef_version = "11.6.0" - + config.berkshelf.enabled = true + config.omnibus.chef_version = :latest + config.vm.network "forwarded_port", guest: 8888, host: 9999 config.vm.provision :chef_solo do |chef| - #Run apt-get update before anything else (specifically postgresql).. + # run apt-get update before anything else (specifically postgresql).. chef.add_recipe "apt" - chef.add_recipe "redash" chef.add_recipe "postgresql" - + chef.add_recipe "redash::redash_pg_schema" + chef.add_recipe "redash::redash" + chef.json = { - "apt" => {"compiletime" => true}, - "postgresql" => {"password" => {"postgres" => "securepass"}}, - "redash" => {"db" => {"host" => "localhost"}}, - } - + "apt" => { "compiletime" => true }, + "postgresql" => { "password" => { "postgres" => __pg_password } }, + "redash" => { "db" => { "host" => "localhost", + "user" => "postgres", + "password" => __pg_password } } + } end end From c6bf7511e176f6e424316534ff15e8dbcd955b0f Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Mon, 25 Nov 2013 10:54:24 +0200 Subject: [PATCH 081/540] Improve Vagrantfile --- vagrant/Vagrantfile | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 8e762be6f..0886d19b2 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -2,35 +2,45 @@ # vi: set ft=ruby : # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" +VAGRANTFILE_API_VERSION = '2' # TODO: should be result of encryption of readable password -__pg_password = "securepass" +POSTGRES_PASSWORD = 'securepass' + +# After starting the vagrant machine, the application is accessible via the URL +# http://localhost:9999 +HOST_PORT_TO_FORWARD_TO_REDASH = 9999 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu-precise-cloudimg-amd64" - config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box" - + config.vm.box = 'ubuntu-precise-cloudimg-amd64' + config.vm.box_url = 'http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box' + + if config.respond_to? :cache + config.cache.auto_detect = true + end + config.berkshelf.enabled = true config.omnibus.chef_version = :latest - config.vm.network "forwarded_port", guest: 8888, host: 9999 + config.vm.network 'forwarded_port', guest: 8888, host: HOST_PORT_TO_FORWARD_TO_REDASH config.vm.provision :chef_solo do |chef| # run apt-get update before anything else (specifically postgresql).. - chef.add_recipe "apt" - chef.add_recipe "postgresql" - chef.add_recipe "redash::redash_pg_schema" - chef.add_recipe "redash::redash" - + chef.add_recipe 'apt' + chef.add_recipe 'postgresql' + chef.add_recipe 'redash::redash_pg_schema' + chef.add_recipe 'redash::redash' + # chef.log_level = :debug chef.json = { - "apt" => { "compiletime" => true }, - "postgresql" => { "password" => { "postgres" => __pg_password } }, - "redash" => { "db" => { "host" => "localhost", - "user" => "postgres", - "password" => __pg_password } } + 'apt' => { 'compiletime' => true }, + 'postgresql' => { 'password' => {'postgres' => POSTGRES_PASSWORD } }, + 'redash' => { 'db' => {'host' => 'localhost', + 'user' => 'postgres', + 'password' => POSTGRES_PASSWORD }, + 'allow' => {'google_app_domain' => 'gmail.com', + 'admins' => ['joe@egmail.com','jack@gmail.com']}} } end end From 75f1ab6183020b43edc078193f2ea235fd08d77d Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Wed, 27 Nov 2013 18:37:58 +0200 Subject: [PATCH 082/540] Force postgresql::server in Vagrant --- vagrant/Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 0886d19b2..ee2098867 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -29,7 +29,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision :chef_solo do |chef| # run apt-get update before anything else (specifically postgresql).. chef.add_recipe 'apt' - chef.add_recipe 'postgresql' + chef.add_recipe 'postgresql::client' + chef.add_recipe 'postgresql::server' chef.add_recipe 'redash::redash_pg_schema' chef.add_recipe 'redash::redash' # chef.log_level = :debug From 7315d9671dc08b5bb8401cf4f252b9a71597d98a Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Wed, 27 Nov 2013 18:46:05 +0200 Subject: [PATCH 083/540] Use md5 digest of readable pg password postgres user --- vagrant/Vagrantfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index ee2098867..ba46dce8d 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -4,9 +4,9 @@ # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = '2' - -# TODO: should be result of encryption of readable password -POSTGRES_PASSWORD = 'securepass' +require "digest/md5" +POSTGRES_PASSWORD = 'securepass' +postgres_password_md5 = Digest::MD5.hexdigest(POSTGRES_PASSWORD) # After starting the vagrant machine, the application is accessible via the URL # http://localhost:9999 @@ -36,7 +36,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # chef.log_level = :debug chef.json = { 'apt' => { 'compiletime' => true }, - 'postgresql' => { 'password' => {'postgres' => POSTGRES_PASSWORD } }, + 'postgresql' => { 'password' => {'postgres' => postgres_password_md5 } }, 'redash' => { 'db' => {'host' => 'localhost', 'user' => 'postgres', 'password' => POSTGRES_PASSWORD }, From a6ab94113f554455ba0e05c8ccbfe15a60e5e961 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Thu, 28 Nov 2013 10:55:28 +0200 Subject: [PATCH 084/540] Correct postgresql encrypted vs cleartext passwords --- vagrant/Vagrantfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index ba46dce8d..e0b188981 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -4,9 +4,13 @@ # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = '2' -require "digest/md5" POSTGRES_PASSWORD = 'securepass' -postgres_password_md5 = Digest::MD5.hexdigest(POSTGRES_PASSWORD) + +# Currently, chef postgress cookbook works with cleartext paswords, +# unless the password begins with 'md5' +# See https://github.com/hw-cookbooks/postgresql/issues/95 +require "digest/md5" +postgres_password_md5 = 'md5'+Digest::MD5.hexdigest(POSTGRES_PASSWORD+'postgres') # After starting the vagrant machine, the application is accessible via the URL # http://localhost:9999 @@ -29,6 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision :chef_solo do |chef| # run apt-get update before anything else (specifically postgresql).. chef.add_recipe 'apt' + chef.add_recipe 'redash::redis_for_redash' chef.add_recipe 'postgresql::client' chef.add_recipe 'postgresql::server' chef.add_recipe 'redash::redash_pg_schema' From 30c3df0829923afced1f1fb22d70ab0cd3d15c90 Mon Sep 17 00:00:00 2001 From: Timor Raiman Date: Mon, 2 Dec 2013 11:54:56 +0200 Subject: [PATCH 085/540] In Vagrantfile, support skiping tarball download, and graft the current development tree in stead --- vagrant/Vagrantfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index e0b188981..415006f58 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -16,6 +16,9 @@ postgres_password_md5 = 'md5'+Digest::MD5.hexdigest(POSTGRES_PASSWORD+'postgres' # http://localhost:9999 HOST_PORT_TO_FORWARD_TO_REDASH = 9999 +# Deploy direcly the code in parent dir; Don't download a release tarball +live_testing_deployment = true + Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = 'ubuntu-precise-cloudimg-amd64' config.vm.box_url = 'http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box' @@ -29,6 +32,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network 'forwarded_port', guest: 8888, host: HOST_PORT_TO_FORWARD_TO_REDASH + if live_testing_deployment + config.vm.synced_folder "..", "/opt/redash" + end config.vm.provision :chef_solo do |chef| # run apt-get update before anything else (specifically postgresql).. @@ -45,8 +51,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 'redash' => { 'db' => {'host' => 'localhost', 'user' => 'postgres', 'password' => POSTGRES_PASSWORD }, - 'allow' => {'google_app_domain' => 'gmail.com', - 'admins' => ['joe@egmail.com','jack@gmail.com']}} + 'allow' => {'google_app_domain' => 'gmail.com', + 'admins' => ['joe@egmail.com','jack@gmail.com']}, + 'install_tarball' => !live_testing_deployment, + 'user' => live_testing_deployment ? 'vagrant' : 'redash'} } end end From 5db8e66ca6db6c91185ce6c9d07b6641ccd8f902 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 26 Dec 2013 16:19:10 +0200 Subject: [PATCH 086/540] Move vagrant related files to top level --- .gitignore | 4 ++++ vagrant/Berksfile => Berksfile | 0 vagrant/Vagrantfile => Vagrantfile | 0 vagrant/.gitignore | 2 -- 4 files changed, 4 insertions(+), 2 deletions(-) rename vagrant/Berksfile => Berksfile (100%) rename vagrant/Vagrantfile => Vagrantfile (100%) delete mode 100644 vagrant/.gitignore diff --git a/.gitignore b/.gitignore index 3dd65166f..03cbf8cb6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ rd_service/settings.py rd_ui/dist .DS_Store + +# Vagrant related +.vagrant +Berksfile.lock diff --git a/vagrant/Berksfile b/Berksfile similarity index 100% rename from vagrant/Berksfile rename to Berksfile diff --git a/vagrant/Vagrantfile b/Vagrantfile similarity index 100% rename from vagrant/Vagrantfile rename to Vagrantfile diff --git a/vagrant/.gitignore b/vagrant/.gitignore deleted file mode 100644 index 43c013aee..000000000 --- a/vagrant/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.vagrant -Berksfile.lock From b4b61f9eb69a45d8d746da84a815ea920968e317 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 4 Jan 2014 10:52:21 +0200 Subject: [PATCH 087/540] Fix cookbook path --- Berksfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Berksfile b/Berksfile index 91560b352..acc59cb36 100644 --- a/Berksfile +++ b/Berksfile @@ -1,4 +1,3 @@ cookbook 'apt' cookbook 'postgresql' -#cookbook 'redash', git: 'git@gitlab.doit9.com:ops/chef-redash.git' -cookbook 'redash', path: '../../chef-redash' +cookbook 'redash', git: 'git@github.com:EverythingMe/chef-redash.git' From efe75c41341de5939eb368a68d6e5383f812f2e0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 4 Jan 2014 10:55:40 +0200 Subject: [PATCH 088/540] Add BaseAuthenticatedHandler and move authentication logic there. --- rd_service/server.py | 54 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index 416f7514e..7d9d39599 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -44,10 +44,6 @@ class BaseHandler(tornado.web.RequestHandler): user = self.get_secure_cookie("user") return user - @tornado.web.authenticated - def prepare(self): - pass - def write_json(self, response, encode=True): if encode: response = json.dumps(response, cls=utils.JSONEncoder) @@ -55,6 +51,12 @@ class BaseHandler(tornado.web.RequestHandler): self.write(response) +class BaseAuthenticatedHandler(BaseHandler): + @tornado.web.authenticated + def prepare(self): + pass + + class PingHandler(tornado.web.RequestHandler): def get(self): self.write("PONG") @@ -79,7 +81,7 @@ class GoogleLoginHandler(tornado.web.RequestHandler, self.authenticate_redirect() -class MainHandler(BaseHandler): +class MainHandler(BaseAuthenticatedHandler): def get(self, *args): email_md5 = hashlib.md5(self.current_user.lower()).hexdigest() gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5 @@ -93,7 +95,7 @@ class MainHandler(BaseHandler): self.render("index.html", user=json.dumps(user)) -class QueryFormatHandler(BaseHandler): +class QueryFormatHandler(BaseAuthenticatedHandler): def post(self): arguments = json.loads(self.request.body) query = arguments.get("query", "") @@ -101,7 +103,7 @@ class QueryFormatHandler(BaseHandler): self.write(sqlparse.format(query, reindent=True, keyword_case='upper')) -class StatusHandler(BaseHandler): +class StatusHandler(BaseAuthenticatedHandler): def get(self): status = {} info = self.redis_connection.info() @@ -122,7 +124,7 @@ class StatusHandler(BaseHandler): self.write_json(status) -class WidgetsHandler(BaseHandler): +class WidgetsHandler(BaseAuthenticatedHandler): def post(self, widget_id=None): widget_properties = json.loads(self.request.body) widget_properties['options'] = json.dumps(widget_properties['options']) @@ -162,7 +164,7 @@ class WidgetsHandler(BaseHandler): widget.delete() -class DashboardHandler(BaseHandler): +class DashboardHandler(BaseAuthenticatedHandler): def get(self, dashboard_slug=None): if dashboard_slug: dashboard = data.models.Dashboard.objects.prefetch_related('widgets__query__latest_query_data').get(slug=dashboard_slug) @@ -195,7 +197,7 @@ class DashboardHandler(BaseHandler): dashboard.save() -class QueriesHandler(BaseHandler): +class QueriesHandler(BaseAuthenticatedHandler): def post(self, id=None): query_def = json.loads(self.request.body) if 'created_at' in query_def: @@ -226,7 +228,7 @@ class QueriesHandler(BaseHandler): self.write_json([q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()]) -class QueryResultsHandler(BaseHandler): +class QueryResultsHandler(BaseAuthenticatedHandler): def get(self, query_result_id): query_result = self.data_manager.get_query_result_by_id(query_result_id) if query_result: @@ -249,21 +251,7 @@ class QueryResultsHandler(BaseHandler): self.write({'job': job.to_dict()}) -class JobsHandler(BaseHandler): - def get(self, job_id=None): - if job_id: - # TODO: if finished, include the query result - job = data.Job.load(self.data_manager.redis_connection, job_id) - self.write({'job': job.to_dict()}) - else: - raise NotImplemented - - def delete(self, job_id): - job = data.Job.load(self.data_manager.redis_connection, job_id) - job.cancel() - - -class CsvQueryResultsHandler(BaseHandler): +class CsvQueryResultsHandler(BaseAuthenticatedHandler): def get(self, query_result_id): query_result = self.data_manager.get_query_result_by_id(query_result_id) if query_result: @@ -286,6 +274,20 @@ class CsvQueryResultsHandler(BaseHandler): self.send_error(404) +class JobsHandler(BaseAuthenticatedHandler): + def get(self, job_id=None): + if job_id: + # TODO: if finished, include the query result + job = data.Job.load(self.data_manager.redis_connection, job_id) + self.write({'job': job.to_dict()}) + else: + raise NotImplemented + + def delete(self, job_id): + job = data.Job.load(self.data_manager.redis_connection, job_id) + job.cancel() + + def get_application(static_path, is_debug, redis_connection, data_manager): return tornado.web.Application([(r"/", MainHandler), (r"/ping", PingHandler), From 52888c47246ec02da6662949fcf282deeef58293 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 4 Jan 2014 11:09:13 +0200 Subject: [PATCH 089/540] Add api_key field to queries. --- rd_service/data/models.py | 10 ++++++++++ rd_service/data/tables.sql | 1 + 2 files changed, 11 insertions(+) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index 9559e8ad6..58c9e9850 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -1,7 +1,9 @@ """ Django ORM based models to describe the data model of re:dash. """ +import hashlib import json +import time from django.db import models from django.template.defaultfilters import slugify import utils @@ -40,6 +42,7 @@ class Query(models.Model): description = models.CharField(max_length=4096) query = models.TextField() query_hash = models.CharField(max_length=32) + api_key = models.CharField(max_length=40) ttl = models.IntegerField() user = models.CharField(max_length=360) created_at = models.DateTimeField(auto_now_add=True) @@ -58,6 +61,7 @@ class Query(models.Model): 'query_hash': self.query_hash, 'ttl': self.ttl, 'user': self.user, + 'api_key': self.api_key, 'created_at': self.created_at, } @@ -92,8 +96,14 @@ LEFT OUTER JOIN def save(self, *args, **kwargs): self.query_hash = utils.gen_query_hash(self.query) + self._set_api_key() super(Query, self).save(*args, **kwargs) + def _set_api_key(self): + if not self.api_key: + self.api_key = hashlib.sha1( + u''.join([str(time.time()), self.query, self.user, self.name])).hexdigest() + def __unicode__(self): return unicode(self.id) diff --git a/rd_service/data/tables.sql b/rd_service/data/tables.sql index b2b77a692..fa6b941ab 100644 --- a/rd_service/data/tables.sql +++ b/rd_service/data/tables.sql @@ -15,6 +15,7 @@ CREATE TABLE "queries" ( "description" varchar(4096), "query" text NOT NULL, "query_hash" varchar(32) NOT NULL, + "api_key" varchar(40), "ttl" integer NOT NULL, "user" varchar(360) NOT NULL, "created_at" timestamp with time zone NOT NULL From b1f9995ce20fc59f196ef059ea271127029c78e3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 4 Jan 2014 11:21:52 +0200 Subject: [PATCH 090/540] Allow downloading CSV by query id and not just query_result id. --- rd_service/server.py | 11 ++++++++--- rd_ui/app/scripts/controllers.js | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index 7d9d39599..ffc1b78b8 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -252,8 +252,13 @@ class QueryResultsHandler(BaseAuthenticatedHandler): class CsvQueryResultsHandler(BaseAuthenticatedHandler): - def get(self, query_result_id): - query_result = self.data_manager.get_query_result_by_id(query_result_id) + def get(self, query_id, result_id=None): + if not result_id: + query = data.models.Query.objects.get(pk=query_id) + if query: + result_id = query.latest_query_data_id + + query_result = result_id and self.data_manager.get_query_result_by_id(result_id) if query_result: self.set_header("Content-Type", "text/csv; charset=UTF-8") s = cStringIO.StringIO() @@ -291,10 +296,10 @@ class JobsHandler(BaseAuthenticatedHandler): def get_application(static_path, is_debug, redis_connection, data_manager): return tornado.web.Application([(r"/", MainHandler), (r"/ping", PingHandler), + (r"/api/queries/([0-9]*)/results(?:/([0-9]*))?.csv", CsvQueryResultsHandler), (r"/api/queries/format", QueryFormatHandler), (r"/api/queries(?:/([0-9]*))?", QueriesHandler), (r"/api/query_results(?:/([0-9]*))?", QueryResultsHandler), - (r"/api/query_results/(.*?).csv", CsvQueryResultsHandler), (r"/api/jobs/(.*)", JobsHandler), (r"/api/widgets(?:/([0-9]*))?", WidgetsHandler), (r"/api/dashboards(?:/(.*))?", DashboardHandler), diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index aca4c7862..ea567d091 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -139,7 +139,7 @@ if ($scope.queryResult.getId() == null) { $scope.dataUri = ""; } else { - $scope.dataUri = '/api/query_results/' + $scope.queryResult.getId() + '.csv'; + $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv'; $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv"; } }); From 36e5df00ab6883a76ece695b6520eda35e50b5db Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 4 Jan 2014 11:53:45 +0200 Subject: [PATCH 091/540] Allow downloading CSV for unauthenticated users with api_key. --- rd_service/server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rd_service/server.py b/rd_service/server.py index ffc1b78b8..d74d38dee 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -252,6 +252,17 @@ class QueryResultsHandler(BaseAuthenticatedHandler): class CsvQueryResultsHandler(BaseAuthenticatedHandler): + def get_current_user(self): + user = super(CsvQueryResultsHandler, self).get_current_user() + if not user: + api_key = self.get_argument("api_key", None) + query = data.models.Query.objects.get(pk=self.path_args[0]) + + if query.api_key and query.api_key == api_key: + user = "API-Key=%s" % api_key + + return user + def get(self, query_id, result_id=None): if not result_id: query = data.models.Query.objects.get(pk=query_id) From ba838863bf6912033a5c930b8231c32f3a7f43a1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 11 Jan 2014 20:04:12 +0200 Subject: [PATCH 092/540] Fix: don't show save warning, when switching tabs. --- rd_ui/app/scripts/controllers.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index ea567d091..2f64e4993 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -40,6 +40,10 @@ } $scope.$on('$locationChangeStart', function(event, next, current) { + if (next.split("#")[0] == current.split("#")[0]) { + return; + } + if($scope.queryChanged && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); From 5ba32cfdc1d72de365dff868b8ccbd4145433a8a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 11 Jan 2014 20:04:27 +0200 Subject: [PATCH 093/540] Save query when pressing Cmd+S. --- rd_ui/app/index.html | 2 ++ rd_ui/app/scripts/controllers.js | 7 +++++++ rd_ui/bower.json | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 71b1db3a2..22b3c2deb 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -101,6 +101,8 @@ + + diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 2f64e4993..2014f3678 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -39,6 +39,11 @@ } } + Mousetrap.bindGlobal("meta+s", function(e) { + e.preventDefault(); + $scope.saveQuery(); + }); + $scope.$on('$locationChangeStart', function(event, next, current) { if (next.split("#")[0] == current.split("#")[0]) { return; @@ -48,6 +53,8 @@ !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); } + + Mousetrap.unbind("meta+s"); }); $scope.$parent.pageTitle = "Query Fiddle"; diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 9792acc10..efb967be8 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -17,7 +17,8 @@ "angular-growl": "0.3.1", "pivottable": "git@github.com:arikfr/pivottable.git#master", "cornelius": "git@github.com:restorando/cornelius.git", - "gridster": "0.2.0" + "gridster": "0.2.0", + "mousetrap": "~1.4.6" }, "devDependencies": { "angular-mocks": "~1.0.7", From 447f1fab1f10c8177688e3828ad6cec18fac3308 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 13 Jan 2014 16:52:35 +0200 Subject: [PATCH 094/540] Add the option to specify analytics code to inject into the template. --- rd_service/server.py | 2 +- rd_service/settings_example.py | 1 + rd_ui/app/index.html | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_service/server.py b/rd_service/server.py index d74d38dee..306fbfa08 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -92,7 +92,7 @@ class MainHandler(BaseAuthenticatedHandler): 'name': self.current_user } - self.render("index.html", user=json.dumps(user)) + self.render("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) class QueryFormatHandler(BaseAuthenticatedHandler): diff --git a/rd_service/settings_example.py b/rd_service/settings_example.py index af8c3d04e..01511ec98 100644 --- a/rd_service/settings_example.py +++ b/rd_service/settings_example.py @@ -22,6 +22,7 @@ WORKERS_COUNT = 2 MAX_CONNECTIONS = 3 COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" LOG_LEVEL = "INFO" +ANALYTICS = "" # Configuration of the operational database for the Django models django.conf.settings.configure(DATABASES = { 'default': { diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 22b3c2deb..d6ffd6f8e 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -119,6 +119,8 @@ From deeb113cdd61c5c537b1257570aa37c2e81fc018 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:01:20 +0200 Subject: [PATCH 095/540] Add canEdit function to user object. Checks if currentUser included in the user string the provided object. The reason I check inclusion and not equality, is to support scenario of multiple users. --- rd_ui/app/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index d6ffd6f8e..85acc93a6 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -120,6 +120,10 @@ From f0169978d43719bf5ea809027ec6b0ca9a82d752 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:01:42 +0200 Subject: [PATCH 096/540] Only save query if the user can edit it. --- rd_ui/app/scripts/controllers.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 2014f3678..bbe54c26f 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -41,7 +41,10 @@ Mousetrap.bindGlobal("meta+s", function(e) { e.preventDefault(); - $scope.saveQuery(); + + if (currentUser.canEdit($scope.query)) { + $scope.saveQuery(); + } }); $scope.$on('$locationChangeStart', function(event, next, current) { From 33fec327a96a2df3c63832fd8e9d859866afdbd5 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:02:10 +0200 Subject: [PATCH 097/540] Only unbind save shortcut if we're actually leaving the page. --- rd_ui/app/scripts/controllers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index bbe54c26f..22efe20b2 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -55,9 +55,9 @@ if($scope.queryChanged && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); + } else { + Mousetrap.unbind("meta+s"); } - - Mousetrap.unbind("meta+s"); }); $scope.$parent.pageTitle = "Query Fiddle"; From e928a29b9851d99aa36578f3d046cc64ef6a0b46 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:03:04 +0200 Subject: [PATCH 098/540] Show warning of leaving the page only if the user can edit the query. --- rd_ui/app/scripts/controllers.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 22efe20b2..bb251baab 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -34,7 +34,7 @@ var pristineQuery = null; $window.onbeforeunload = function(){ - if ($scope.queryChanged) { + if (currentUser.canEdit($scope.query) && $scope.queryChanged) { return leavingPageText; } } @@ -52,6 +52,10 @@ return; } + if (!currentUser.canEdit($scope.query)) { + return; + } + if($scope.queryChanged && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); From 4102cab43bfbaa49f578c76291642e183e459c44 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:11:19 +0200 Subject: [PATCH 099/540] use currentUser.canEdit in more places --- rd_ui/app/index.html | 2 +- rd_ui/app/scripts/services/dashboards.js | 2 +- rd_ui/app/views/queryfiddle.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 85acc93a6..37cdbaecc 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -121,7 +121,7 @@ var currentUser = {% raw user %}; currentUser.canEdit = function(object) { - return object.user.indexOf(currentUser.name) != -1; + return object.user && (object.user.indexOf(currentUser.name) != -1); } {% raw analytics %} diff --git a/rd_ui/app/scripts/services/dashboards.js b/rd_ui/app/scripts/services/dashboards.js index 890c55c6f..1eb0c08e3 100644 --- a/rd_ui/app/scripts/services/dashboards.js +++ b/rd_ui/app/scripts/services/dashboards.js @@ -2,7 +2,7 @@ var Dashboard = function($resource) { var resource = $resource('/api/dashboards/:slug', {slug: '@slug'}); resource.prototype.canEdit = function() { - return currentUser.is_admin || currentUser.name == this.user; + return currentUser.is_admin || currentUser.canEdit(this); } return resource; } diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 3614ddb1a..87944a35b 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -17,7 +17,7 @@
    - From a35df2aed8e9e801e3930f4589c9ed5b69e026eb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:24:17 +0200 Subject: [PATCH 100/540] Fix: homepage was skipping dashboards in other group --- rd_ui/app/scripts/controllers.js | 6 +++--- rd_ui/app/views/index.html | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index bb251baab..f5ebf00e3 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -324,15 +324,15 @@ $scope.reloadDashboards = function() { Dashboard.query(function (dashboards) { $scope.dashboards = _.sortBy(dashboards, "name"); - $scope.groupedDashboards = _.groupBy($scope.dashboards, function(d) { + $scope.allDashboards = _.groupBy($scope.dashboards, function(d) { parts = d.name.split(":"); if (parts.length == 1) { return "Other"; } return parts[0]; }); - $scope.otherDashboards = $scope.groupedDashboards['Other'] || []; - delete $scope.groupedDashboards['Other']; + $scope.otherDashboards = $scope.allDashboards['Other'] || []; + $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other'); }); } diff --git a/rd_ui/app/views/index.html b/rd_ui/app/views/index.html index 2f7342043..8db531646 100644 --- a/rd_ui/app/views/index.html +++ b/rd_ui/app/views/index.html @@ -1,6 +1,6 @@

    Dashboards

    -
    +
    {{name}} From c16c0dd454a93c57e04133b15989d2e33777de39 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 14 Jan 2014 09:32:23 +0200 Subject: [PATCH 101/540] Add semicolon --- rd_ui/app/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 37cdbaecc..e56c886a1 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -122,7 +122,7 @@ currentUser.canEdit = function(object) { return object.user && (object.user.indexOf(currentUser.name) != -1); - } + }; {% raw analytics %} From 7e197d2a574a79c7fb26c195a7c016d1a36254b4 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 7 Jan 2014 11:28:13 +0200 Subject: [PATCH 102/540] upgrade: angular v1.2.7 --- .gitignore | 1 + rd_ui/app/index.html | 1 + rd_ui/app/scripts/app.js | 5 +++-- rd_ui/app/scripts/controllers.js | 2 +- rd_ui/bower.json | 5 +++-- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 03cbf8cb6..059ee4bdd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ rd_ui/dist # Vagrant related .vagrant Berksfile.lock +rd_service/dump.rdb diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index e56c886a1..d4877bbbb 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -83,6 +83,7 @@ + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 9835d089b..04718e989 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -11,11 +11,12 @@ angular.module('redash', [ 'angularMoment', 'ui.bootstrap', 'smartTable.table', - 'ngResource' + 'ngResource', + 'ngRoute' ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', function($routeProvider, $locationProvider, $compileProvider, growlProvider) { - $compileProvider.urlSanitizationWhitelist(/^\s*(https?|http|data):/); + $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/); $locationProvider.html5Mode(true); growlProvider.globalTimeToLive(2000); diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index f5ebf00e3..4a62a1096 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -316,7 +316,7 @@ } filterQueries(); - }) + }); } var MainCtrl = function ($scope, Dashboard, notifications) { diff --git a/rd_ui/bower.json b/rd_ui/bower.json index efb967be8..321f39913 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -2,7 +2,7 @@ "name": "rdUi", "version": "0.1.0", "dependencies": { - "angular": "1.0.8", + "angular": "1.2.7", "json3": "3.2.4", "jquery": "1.9.1", "bootstrap": "3.0.0", @@ -15,6 +15,7 @@ "underscore": "1.5.1", "angular-resource": "1.0.7", "angular-growl": "0.3.1", + "angular-route": "1.2.7", "pivottable": "git@github.com:arikfr/pivottable.git#master", "cornelius": "git@github.com:restorando/cornelius.git", "gridster": "0.2.0", @@ -25,7 +26,7 @@ "angular-scenario": "~1.0.7" }, "resolutions": { - "angular": "1.0.8", + "angular": "~1.2.7", "jquery": "~1.9.1" } } From 4a1563365db1938e5bd8ae4b4134ad066ad5fa61 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 11 Jan 2014 14:50:49 +0200 Subject: [PATCH 103/540] Fix: default predicate should be undefined and not empty string --- rd_ui/app/scripts/smart-table.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/smart-table.js b/rd_ui/app/scripts/smart-table.js index 33249a5e6..a4706602f 100644 --- a/rd_ui/app/scripts/smart-table.js +++ b/rd_ui/app/scripts/smart-table.js @@ -15,7 +15,7 @@ sortPredicate: '', formatFunction: '', formatParameter: '', - filterPredicate: '', + filterPredicate: undefined, cellTemplateUrl: '', headerClass: '', cellClass: '' @@ -184,7 +184,7 @@ replace: false, link: function (scope, element, attr, ctrl) { - scope.searchValue = ''; + scope.searchValue = undefined; scope.$watch('searchValue', function (value) { //todo perf improvement only filter on blur ? @@ -464,14 +464,13 @@ * @param column */ this.search = function (input, column) { - //update column and global predicate if (column && scope.columns.indexOf(column) !== -1) { predicate.$ = ''; column.filterPredicate = input; } else { for (var j = 0, l = scope.columns.length; j < l; j++) { - scope.columns[j].filterPredicate = ''; + scope.columns[j].filterPredicate = undefined; } predicate.$ = input; } From aa7d066f6ca7162d08066ff893602eba107a5d18 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 14 Jan 2014 16:42:12 +0200 Subject: [PATCH 104/540] Add ability to edit query description (FED #22) --- rd_ui/app/scripts/directives.js | 10 ++++++++-- rd_ui/app/styles/redash.css | 19 +++++++++++++------ rd_ui/app/views/queryfiddle.html | 15 ++++++++++++++- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 9d739b46d..d624abed3 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -162,10 +162,16 @@ directives.directive('editInPlace', function () { value: '=', ignoreBlanks: '=' }, - template: '', + template: function(tElement, tAttrs) { + var elType = tAttrs.editor || 'input'; + var placeholder = tAttrs.placeholder || 'Click to edit'; + return '' + + '' + placeholder + '' + + '<{elType} ng-model="value">'.replace('{elType}', elType); + }, link: function ($scope, element, attrs) { // Let's get a reference to the input element, as we'll want to reference it. - var inputElement = angular.element(element.children()[1]); + var inputElement = angular.element(element.children()[2]); // This directive should have a set class so we can style it. element.addClass('edit-in-place'); diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index e935e6b33..1abdcd40b 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -23,19 +23,26 @@ a.navbar-brand { } .edit-in-place span { - cursor: pointer; + cursor: pointer; } -.edit-in-place input { - display: none; +.edit-in-place input, +.edit-in-place textarea { + display: none; +} + +.edit-in-place textarea { + height: 80px; + width: 250px; } .edit-in-place.active span { - display: none; + display: none; } -.edit-in-place.active input { - display: inline-block; +.edit-in-place.active input, +.edit-in-place.active textarea { + display: inline-block; } .delete-button { diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 87944a35b..81a00f617 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -3,7 +3,20 @@

    - +
    +
    +

    + +

    +
    +
    +
    +
    +

    + +

    +
    +

    From ad97b7402653ab3a442faac23769e19d190a7c14 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 15 Jan 2014 11:15:35 +0200 Subject: [PATCH 105/540] Change look of edit in place text area. --- rd_ui/app/scripts/directives.js | 2 +- rd_ui/app/styles/redash.css | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index d624abed3..80fbf7abd 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -167,7 +167,7 @@ directives.directive('editInPlace', function () { var placeholder = tAttrs.placeholder || 'Click to edit'; return '' + '' + placeholder + '' + - '<{elType} ng-model="value">'.replace('{elType}', elType); + '<{elType} ng-model="value" class="form-control" rows="2">'.replace('{elType}', elType); }, link: function ($scope, element, attrs) { // Let's get a reference to the input element, as we'll want to reference it. diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 1abdcd40b..82a86f894 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -26,16 +26,15 @@ a.navbar-brand { cursor: pointer; } +.edit-in-place span:hover { + background: yellow; +} + .edit-in-place input, .edit-in-place textarea { display: none; } -.edit-in-place textarea { - height: 80px; - width: 250px; -} - .edit-in-place.active span { display: none; } From e028dfe2f9ad8960f52aefe3bcea6efe91ee00d7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 15 Jan 2014 11:38:30 +0200 Subject: [PATCH 106/540] Show edit-in-place only if the user can edit. --- rd_ui/app/views/queryfiddle.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 81a00f617..e8f1dcf91 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -4,17 +4,17 @@

    -
    -

    - -

    +
    +

    + + +

    -
    -

    - -

    +
    + +

    From b8fb6200f0e8f66a0cafb2c0e1d1509e5697fb4e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 15 Jan 2014 11:38:38 +0200 Subject: [PATCH 107/540] Show description in dashboard. --- rd_ui/app/scripts/controllers.js | 8 ++++++-- rd_ui/app/views/dashboard.html | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 4a62a1096..87e05b3f7 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -5,7 +5,7 @@ }); }; - var WidgetCtrl = function ($scope, $http, Query) { + var WidgetCtrl = function ($scope, $http, $location, Query) { $scope.deleteWidget = function() { if (!confirm('Are you sure you want to remove "' + $scope.widget.query.name + '" from the dashboard?')) { return; @@ -18,6 +18,10 @@ }) }); }); + }; + + $scope.open = function(query) { + $location.path('/queries/' + query.id); } $scope.query = new Query($scope.widget.query); @@ -363,7 +367,7 @@ angular.module('redash.controllers', []) .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) - .controller('WidgetCtrl', ['$scope', '$http', 'Query', WidgetCtrl]) + .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) .controller('QueryFiddleCtrl', ['$scope', '$window', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index b16ac3ea3..676ec6d20 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -21,16 +21,11 @@
    -

    - - {{query.name}} - - - +

    +

    + +

    +

    @@ -39,6 +34,13 @@
    +
    + Updated: + + +
    From faad6766a8de596b15ff31ae681027dc674e51a4 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 15 Jan 2014 13:57:22 +0200 Subject: [PATCH 108/540] edit-in-place: editable attr --- rd_ui/app/scripts/directives.js | 7 ++++--- rd_ui/app/styles/redash.css | 6 +++--- rd_ui/app/views/queryfiddle.html | 20 ++++++-------------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 80fbf7abd..c5dc272d6 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -160,13 +160,14 @@ directives.directive('editInPlace', function () { restrict: 'E', scope: { value: '=', - ignoreBlanks: '=' + ignoreBlanks: '=', + editable: '=' }, template: function(tElement, tAttrs) { var elType = tAttrs.editor || 'input'; var placeholder = tAttrs.placeholder || 'Click to edit'; - return '' + - '' + placeholder + '' + + return '' + + '' + placeholder + '' + '<{elType} ng-model="value" class="form-control" rows="2">'.replace('{elType}', elType); }, link: function ($scope, element, attrs) { diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 82a86f894..e5ba6ace3 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -22,12 +22,12 @@ a.navbar-brand { margin-bottom: 5px; } -.edit-in-place span { +.edit-in-place.editable span { cursor: pointer; } -.edit-in-place span:hover { - background: yellow; +.edit-in-place.editable span:hover { + background: #FCFCA2; } .edit-in-place input, diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index e8f1dcf91..f89bcb69f 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -3,21 +3,13 @@

    -
    -
    -

    - - -

    -
    -
    -
    -
    - - -
    -
    +

    + +

    +

    + +

    From 9b557657b1740920687fb6375a76f85bfbf6997e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 15 Jan 2014 14:35:06 +0200 Subject: [PATCH 109/540] Make margin margin after description smaller --- rd_ui/app/styles/redash.css | 4 ++++ rd_ui/app/views/queryfiddle.html | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index e5ba6ace3..6f9da253b 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -52,6 +52,10 @@ a.navbar-brand { color: white; } +.panel-heading > p:last-child { + margin-bottom: 0px; +} + /* angular-growl */ .growl { position: fixed; diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index f89bcb69f..61b037098 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -4,11 +4,17 @@

    - -

    + +

    - +

    From f7d4c285f5dae4a0c4c45ea523482f66d4db0de6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 15 Jan 2014 15:05:19 +0200 Subject: [PATCH 110/540] Show link button on dashboard --- rd_ui/app/views/dashboard.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 676ec6d20..fa9c5b0c4 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -39,7 +39,11 @@ tooltip="next update {{nextUpdateTime}} (query runtime: {{queryResult.getRuntime() | durationHumanize}})" tooltip-placement="bottom">Updated: - + + + + +
    From 9b6b6a6cd77fbfaf9e76a31a0eebc7f3a36aee2d Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 15 Jan 2014 16:18:10 +0200 Subject: [PATCH 111/540] edit-in-place: set ng-class inside directive --- rd_ui/app/scripts/directives.js | 4 ++-- rd_ui/app/styles/redash.css | 4 ++-- rd_ui/app/views/queryfiddle.html | 12 +++--------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index c5dc272d6..a49e0476a 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -166,8 +166,8 @@ directives.directive('editInPlace', function () { template: function(tElement, tAttrs) { var elType = tAttrs.editor || 'input'; var placeholder = tAttrs.placeholder || 'Click to edit'; - return '' + - '' + placeholder + '' + + return '' + + '' + placeholder + '' + '<{elType} ng-model="value" class="form-control" rows="2">'.replace('{elType}', elType); }, link: function ($scope, element, attrs) { diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 6f9da253b..843ccff76 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -22,11 +22,11 @@ a.navbar-brand { margin-bottom: 5px; } -.edit-in-place.editable span { +.edit-in-place span.editable { cursor: pointer; } -.edit-in-place.editable span:hover { +.edit-in-place span.editable:hover { background: #FCFCA2; } diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 61b037098..7ab0b32d9 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -4,17 +4,11 @@

    - -

    + +

    - +

    From 01e393cf8c5b821f7f025c44eb676fa3404c8262 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 16 Jan 2014 13:44:13 +0200 Subject: [PATCH 112/540] use ngForm in queryFiddle page for detecting pristine and dirty states --- rd_ui/app/scripts/controllers.js | 15 ++---- rd_ui/app/views/queryfiddle.html | 90 ++++++++++++++++---------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 87e05b3f7..923b6fb85 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -35,10 +35,9 @@ var QueryFiddleCtrl = function ($scope, $window, $routeParams, $http, $location, growl, notifications, Query) { var leavingPageText = "You will lose your changes if you leave"; - var pristineQuery = null; $window.onbeforeunload = function(){ - if (currentUser.canEdit($scope.query) && $scope.queryChanged) { + if (currentUser.canEdit($scope.query) && $scope.queryForm.$dirty) { return leavingPageText; } } @@ -60,7 +59,7 @@ return; } - if($scope.queryChanged && + if($scope.queryForm.$dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); } else { @@ -92,8 +91,7 @@ } delete $scope.query.latest_query_data; $scope.query.$save(function (q) { - pristineQuery = q.query; - $scope.queryChanged = false; + $scope.queryForm.$setPristine(); if (duplicate) { growl.addInfoMessage("Query duplicated.", {ttl: 2000}); @@ -186,7 +184,7 @@ if ($routeParams.queryId != undefined) { $scope.query = Query.get({id: $routeParams.queryId}, function(q) { - pristineQuery = q.query; + $scope.queryForm.$setPristine(); $scope.queryResult = $scope.query.getQueryResult(); }); } else { @@ -197,11 +195,6 @@ $scope.$watch('query.name', function() { $scope.$parent.pageTitle = $scope.query.name; }); - $scope.$watch('query.query', function(q) { - if (q) { - $scope.queryChanged = (q != pristineQuery); - } - }); $scope.executeQuery = function() { $scope.queryResult = $scope.query.getQueryResult(0); diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 7ab0b32d9..37a5aeaaf 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -1,53 +1,55 @@
    -
    -
    -

    +
    +
    +
    +

    +

    + +

    +

    - -

    -

    -

    - -

    -
    -
    - - -
    - - Download Data Set - - - -
    - - - -
    + +

    +
    + -
    -
    - Query runtime: {{queryResult.getRuntime() | durationHumanize}} | - Last update time: | - Rows: {{queryResult.getData().length}} | - Created by: {{query.user}} -
    Refresh query:
    -
    -
    +
    + + Download Data Set + + -
    - Executing query... - -
    -
    - Query in queue... - -
    -
    Error running query: {{queryResult.getError()}}
    +
    + + + +
    +
    + +
    +
    + Query runtime: {{queryResult.getRuntime() | durationHumanize}} | + Last update time: | + Rows: {{queryResult.getData().length}} | + Created by: {{query.user}} +
    Refresh query:
    +
    +
    + +
    + Executing query... + +
    +
    + Query in queue... + +
    +
    Error running query: {{queryResult.getError()}}
    +
    From 9ce6b81ae0ddfa4f18cb6223d266719b5875b8ce Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 16 Jan 2014 16:07:51 +0200 Subject: [PATCH 113/540] queryFiddle: reset form state if changes have been reverted to original values (no ngForm) --- rd_ui/app/scripts/controllers.js | 19 ++++++-- rd_ui/app/scripts/services.js | 4 ++ rd_ui/app/views/queryfiddle.html | 82 ++++++++++++++++---------------- 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 923b6fb85..957f011cb 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -34,10 +34,13 @@ } var QueryFiddleCtrl = function ($scope, $window, $routeParams, $http, $location, growl, notifications, Query) { + var pristineHash = null; + $scope.dirty = undefined; + var leavingPageText = "You will lose your changes if you leave"; $window.onbeforeunload = function(){ - if (currentUser.canEdit($scope.query) && $scope.queryForm.$dirty) { + if (currentUser.canEdit($scope.query) && $scope.dirty) { return leavingPageText; } } @@ -59,7 +62,7 @@ return; } - if($scope.queryForm.$dirty && + if($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { event.preventDefault(); } else { @@ -91,7 +94,8 @@ } delete $scope.query.latest_query_data; $scope.query.$save(function (q) { - $scope.queryForm.$setPristine(); + pristineHash = q.getHash(); + $scope.dirty = false; if (duplicate) { growl.addInfoMessage("Query duplicated.", {ttl: 2000}); @@ -184,7 +188,8 @@ if ($routeParams.queryId != undefined) { $scope.query = Query.get({id: $routeParams.queryId}, function(q) { - $scope.queryForm.$setPristine(); + pristineHash = q.getHash(); + $scope.dirty = false; $scope.queryResult = $scope.query.getQueryResult(); }); } else { @@ -196,6 +201,12 @@ $scope.$parent.pageTitle = $scope.query.name; }); + $scope.$watch(function() { + return $scope.query.getHash(); + }, function(newHash) { + $scope.dirty = (newHash !== pristineHash); + }); + $scope.executeQuery = function() { $scope.queryResult = $scope.query.getQueryResult(0); $scope.lockButton(true); diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index ab32673ce..230d2d7e1 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -275,6 +275,10 @@ return queryResult; } + Query.prototype.getHash = function() { + return [this.name, this.description, this.query].join('!#'); + } + return Query; } diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 37a5aeaaf..bcdcec8d4 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -1,55 +1,53 @@
    -
    -
    -
    -

    -

    - -

    -

    +
    +
    +

    - -

    -

    -
    - + +

    + +

    + +

    +
    +
    + -
    - - Download Data Set - - +
    + + Download Data Set + + -
    - - - -
    +
    + + +
    -
    -
    - Query runtime: {{queryResult.getRuntime() | durationHumanize}} | - Last update time: | - Rows: {{queryResult.getData().length}} | - Created by: {{query.user}} -
    Refresh query:
    -
    -
    -
    - Executing query... -
    -
    - Query in queue... - +
    + Query runtime: {{queryResult.getRuntime() | durationHumanize}} | + Last update time: | + Rows: {{queryResult.getData().length}} | + Created by: {{query.user}} +
    Refresh query:
    -
    Error running query: {{queryResult.getError()}}
    - +
    + +
    + Executing query... + +
    +
    + Query in queue... + +
    +
    Error running query: {{queryResult.getError()}}
    From 0f5b4887eef121b3cad8d602ea88a998a8b536ec Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jan 2014 08:56:54 +0200 Subject: [PATCH 114/540] Option to control chart type --- rd_ui/app/scripts/query_fiddle/renderers.js | 7 +++++-- rd_ui/app/views/dashboard.html | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 36caa05ca..0d92a60fa 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -84,7 +84,9 @@ renderers.directive('chartRenderer', function () { restrict: 'E', scope: { queryResult: '=', - stacking: '&' + stacking: '&', + options: '=?', + type: '&' }, template: "", replace: false, @@ -103,8 +105,9 @@ renderers.directive('chartRenderer', function () { stacking = 'normal'; } + var chart_type = $scope.type() || 'column'; _.each($scope.queryResult.getChartData(), function (s) { - $scope.chartSeries.push(_.extend(s, {'stacking': stacking})); + $scope.chartSeries.push(_.extend(s, {'stacking': stacking, 'type': chart_type}, $scope.options)); }); } }); diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index fa9c5b0c4..f19e8dd33 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -30,7 +30,7 @@
    - +
    From d4e484fd826c0c07fa6e935e26991f78f87c1fbd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jan 2014 09:27:01 +0200 Subject: [PATCH 115/540] Pass the widget options object as is. --- rd_ui/app/scripts/query_fiddle/renderers.js | 12 ++---------- rd_ui/app/views/dashboard.html | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 0d92a60fa..bb2fd29f5 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -84,9 +84,7 @@ renderers.directive('chartRenderer', function () { restrict: 'E', scope: { queryResult: '=', - stacking: '&', - options: '=?', - type: '&' + options: '=?' }, template: "", replace: false, @@ -100,14 +98,8 @@ renderers.directive('chartRenderer', function () { } else { $scope.chartSeries.splice(0, $scope.chartSeries.length); - var stacking = null; - if ($scope.stacking() === undefined) { - stacking = 'normal'; - } - - var chart_type = $scope.type() || 'column'; _.each($scope.queryResult.getChartData(), function (s) { - $scope.chartSeries.push(_.extend(s, {'stacking': stacking, 'type': chart_type}, $scope.options)); + $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}, $scope.options)); }); } }); diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index f19e8dd33..6892233c1 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -30,7 +30,7 @@
    - +
    From 441f9c677a9339816d1cfa2566ed10bafc8eec32 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 23 Jan 2014 14:57:08 +0200 Subject: [PATCH 116/540] QueryFiddle: reset sorting when executing query. fixes #33 --- rd_ui/app/scripts/query_fiddle/renderers.js | 2 +- rd_ui/app/scripts/smart-table.js | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index bb2fd29f5..2fca58a1f 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -123,7 +123,7 @@ renderers.directive('gridRenderer', function () { isPaginationEnabled: true, itemsByPage: $scope.itemsPerPage || 15, maxSize: 8 - } + }; $scope.$watch('queryResult && queryResult.getData()', function (data) { if (!data) { diff --git a/rd_ui/app/scripts/smart-table.js b/rd_ui/app/scripts/smart-table.js index a4706602f..770a64442 100644 --- a/rd_ui/app/scripts/smart-table.js +++ b/rd_ui/app/scripts/smart-table.js @@ -113,8 +113,10 @@ //if item are added or removed into the data model from outside the grid scope.$watch('dataCollection', function (oldValue, newValue) { - if (oldValue !== newValue) { - ctrl.sortBy();//it will trigger the refresh... some hack ? + // evme: + // reset sorting when data updates (executing query again) + if (newValue) { + ctrl.resetSort(); } }); @@ -496,6 +498,12 @@ return scope.isPaginationEnabled ? arrayUtility.fromTo(output, (scope.currentPage - 1) * scope.itemsByPage, scope.itemsByPage) : output; }; + this.resetSort = function() { + lastColumnSort = null; + predicate = {}; + this.sortBy(); + }; + /*//////////// Column API ///////////*/ From 2b13ef1063af8b9faa4f8482e9c2fdd15bd1077a Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 23 Jan 2014 18:12:44 +0200 Subject: [PATCH 117/540] Dashboard: update layout editor when adding/removing widgets. fixes #9 --- rd_ui/app/scripts/directives.js | 73 ++++++++++++++++++----------- rd_ui/app/views/edit_dashboard.html | 14 ++---- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a49e0476a..92cb96060 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -33,35 +33,52 @@ directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Da templateUrl: '/views/edit_dashboard.html', replace: true, link: function($scope, element, attrs) { - $scope.$watch('dashboard.widgets', function() { - if ($scope.dashboard.widgets) { - $scope.layout = []; - _.each($scope.dashboard.widgets, function(row, rowIndex) { - _.each(row, function(widget, colIndex) { - $scope.layout.push({ - id: widget.id, - col: colIndex+1, - row: rowIndex+1, - ySize: 1, - xSize: widget.width, - name: widget.query.name - }) - }) - }); - - $timeout(function () { - $(".gridster ul").gridster({ - widget_margins: [5, 5], - widget_base_dimensions: [260, 100], - min_cols: 2, - max_cols: 2, - serialize_params: function ($w, wgd) { - return { col: wgd.col, row: wgd.row, id: $w.data('widget-id') } - } - }); - }); + var gridster = element.find(".gridster ul").gridster({ + widget_margins: [5, 5], + widget_base_dimensions: [260, 100], + min_cols: 2, + max_cols: 2, + serialize_params: function($w, wgd) { + return { + col: wgd.col, + row: wgd.row, + id: $w.data('widget-id') + } } - }); + }).data('gridster'); + + var gsItemTemplate = '
  • ' + + '
    {name}' + + '
    • '; + + $scope.$watch('dashboard.widgets', function(widgets) { + $timeout(function () { + gridster.remove_all_widgets(); + + if (widgets && widgets.length) { + var layout = []; + + _.each(widgets, function(row, rowIndex) { + _.each(row, function(widget, colIndex) { + layout.push({ + id: widget.id, + col: colIndex+1, + row: rowIndex+1, + ySize: 1, + xSize: widget.width, + name: widget.query.name + }); + }); + }); + + _.each(layout, function(item) { + var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); + gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); + + }); + } + }); + }, true); $scope.saveDashboard = function() { $scope.saveInProgress = true; diff --git a/rd_ui/app/views/edit_dashboard.html b/rd_ui/app/views/edit_dashboard.html index f776b0356..77020d99c 100644 --- a/rd_ui/app/views/edit_dashboard.html +++ b/rd_ui/app/views/edit_dashboard.html @@ -10,17 +10,9 @@

    -

    -

    -
      -
    • -
      {{widget.name}}
      -
      • -
    -
    -

    +
    +
      +
      From 08d02838d3476479a6b6cc7450db4965d56971c7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 25 Jan 2014 21:24:31 +0200 Subject: [PATCH 118/540] CircleCI configuration & makefile. --- Makefile | 11 +++++++++++ circle.yml | 10 ++++++++++ rd_ui/package.json | 1 - 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 circle.yml diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..3458655bc --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +NAME=redash +FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME)."$(CIRCLE_BUILD_NUM)".tar.gz + +deps: + cd rd_ui && npm install + cd rd_ui && npm install grunt-cli bower + cd rd_ui && bower install + cd rd_ui && grunt build + +pack: + tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * diff --git a/circle.yml b/circle.yml new file mode 100644 index 000000000..1e68fc585 --- /dev/null +++ b/circle.yml @@ -0,0 +1,10 @@ +machine: + node: + version: + 0.10.22 +dependencies: + pre: + - make deps +test: + post: + - make pack diff --git a/rd_ui/package.json b/rd_ui/package.json index f703c69eb..cf2ca05ae 100644 --- a/rd_ui/package.json +++ b/rd_ui/package.json @@ -2,7 +2,6 @@ "name": "rd-ui", "version": "0.1.0", "dependencies": { - "grunt-karma": "~0.6.1" }, "devDependencies": { "grunt": "git+https://github.com/gruntjs/grunt.git#08a3af5", From 95759addda81ae0d5e4146094b22d65500b78a3d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:26:32 +0200 Subject: [PATCH 119/540] Upload version to github script --- bin/upload_version.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 bin/upload_version.py diff --git a/bin/upload_version.py b/bin/upload_version.py new file mode 100644 index 000000000..4c01906b1 --- /dev/null +++ b/bin/upload_version.py @@ -0,0 +1,31 @@ +#!python +import os +import sys +import json +import requests + +if __name__ == '__main__': + # create release + version = sys.argv[1] + filename = sys.argv[2] + github_token = os.environ['GITHUB_TOKEN'] + auth = (github_token, 'x-oauth-basic') + commit_sha = os.environ['CIRCLE_SHA1'] + + params = json.dumps({ + 'tag_name': 'v{0}'.format(version), + 'name': 're:dash v{0}'.format(version), + 'target_commitish': commit_sha + }) + + response = requests.post('https://api.github.com/repos/everythingme/redash/releases', + data=params, + auth=auth) + + upload_url = response.json()['upload_url'] + upload_url = upload_url.replace('{?name}', '') + + with open(filename) as file_content: + headers = {'Content-Type': 'application/gzip'} + response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False) + From 2a9374e10fa413123db1717bd5c17c1a429db4db Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:26:46 +0200 Subject: [PATCH 120/540] Add version to gzip file --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3458655bc..34d7d9fec 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ NAME=redash -FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME)."$(CIRCLE_BUILD_NUM)".tar.gz +VERSION=0.1 +FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).$(CIRCLE_BUILD_NUM).tar.gz deps: cd rd_ui && npm install From e7145d9d481c4e8f5d370b80d2d9cdb047a16396 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:30:16 +0200 Subject: [PATCH 121/540] Cirlce: upload file to GitHub when done. --- Makefile | 6 +++++- circle.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 34d7d9fec..ddd53e6d0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ NAME=redash VERSION=0.1 -FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).$(CIRCLE_BUILD_NUM).tar.gz +FULL_VERSION=$(VERSION).$(CIRCLE_BUILD_NUM) +FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz deps: cd rd_ui && npm install @@ -10,3 +11,6 @@ deps: pack: tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * + +upload: + python bin/upload_version.py $(FULL_VERSION) $(FILENAME) diff --git a/circle.yml b/circle.yml index 1e68fc585..955cb5a02 100644 --- a/circle.yml +++ b/circle.yml @@ -8,3 +8,8 @@ dependencies: test: post: - make pack +deployment: + github: + branch: master + commands: + - make upload From ed361838ab263cfc74fce7876d7914fb6f6d102f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:34:31 +0200 Subject: [PATCH 122/540] Install requests in CircleCI --- circle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/circle.yml b/circle.yml index 955cb5a02..d57a76fa3 100644 --- a/circle.yml +++ b/circle.yml @@ -5,6 +5,7 @@ machine: dependencies: pre: - make deps + - pip install requests test: post: - make pack From 16534e29328ea872fbb7c5991bc06636c676a605 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:51:07 +0200 Subject: [PATCH 123/540] Set Python version --- circle.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/circle.yml b/circle.yml index d57a76fa3..f39a42a91 100644 --- a/circle.yml +++ b/circle.yml @@ -2,6 +2,9 @@ machine: node: version: 0.10.22 + python: + version: + 2.7 dependencies: pre: - make deps From 2d55f59120ce37c499bf2484e3857684c72e134b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:54:32 +0200 Subject: [PATCH 124/540] More explicit python version --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index f39a42a91..9b4ecfb6b 100644 --- a/circle.yml +++ b/circle.yml @@ -4,7 +4,7 @@ machine: 0.10.22 python: version: - 2.7 + 2.7.3 dependencies: pre: - make deps From 1d28b7901ca9ffaf7f41eb8e298b27b07ca73740 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 16:59:59 +0200 Subject: [PATCH 125/540] Use only filename; without path --- bin/upload_version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/upload_version.py b/bin/upload_version.py index 4c01906b1..8e20090a3 100644 --- a/bin/upload_version.py +++ b/bin/upload_version.py @@ -5,9 +5,8 @@ import json import requests if __name__ == '__main__': - # create release version = sys.argv[1] - filename = sys.argv[2] + filename = sys.argv[2].split('/')[-1] github_token = os.environ['GITHUB_TOKEN'] auth = (github_token, 'x-oauth-basic') commit_sha = os.environ['CIRCLE_SHA1'] From 4d60c735ed649080c863dd334bc876641440ab88 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 26 Jan 2014 17:04:12 +0200 Subject: [PATCH 126/540] Fix to upload script --- bin/upload_version.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/upload_version.py b/bin/upload_version.py index 8e20090a3..ac9bd534f 100644 --- a/bin/upload_version.py +++ b/bin/upload_version.py @@ -6,7 +6,8 @@ import requests if __name__ == '__main__': version = sys.argv[1] - filename = sys.argv[2].split('/')[-1] + filepath = sys.argv[2] + filename = filepath.split('/')[-1] github_token = os.environ['GITHUB_TOKEN'] auth = (github_token, 'x-oauth-basic') commit_sha = os.environ['CIRCLE_SHA1'] @@ -24,7 +25,7 @@ if __name__ == '__main__': upload_url = response.json()['upload_url'] upload_url = upload_url.replace('{?name}', '') - with open(filename) as file_content: + with open(filepath) as file_content: headers = {'Content-Type': 'application/gzip'} response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False) From 87e0962c5a840be92e858e814ebcd1933c760be4 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Wed, 29 Jan 2014 19:02:21 +0200 Subject: [PATCH 127/540] MySQL query runner --- rd_service/data/manager.py | 8 +++-- rd_service/data/query_runner.py | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 81f7d46a2..6854d85fc 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -11,6 +11,7 @@ import redis import time import query_runner import worker +import settings from utils import gen_query_hash @@ -153,8 +154,11 @@ class Manager(object): def start_workers(self, workers_count, connection_string): if self.workers: return self.workers - - runner = query_runner.redshift(connection_string) + + if settings.CONNECTION_ADAPTER == "mysql": + runner = query_runner.mysql(connection_string) + else: + runner = query_runner.redshift(connection_string) self.workers = [worker.Worker(self, runner) for _ in range(workers_count)] for w in self.workers: diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index a39fff8c6..0a23b52e5 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -6,12 +6,70 @@ QueryRunner is the function that the workers use, to execute queries. This is th Because the worker just pass the query, this can be used with any data store that has some sort of query language (for example: HiveQL). """ +import logging import json import psycopg2 +import MySQLdb import sys import select from .utils import JSONEncoder +def mysql(connection_string): + def column_friendly_name(column_name): + return column_name + + def query_runner(query): + # TODO: remove these lines + import settings + connection_string = settings.CONNECTION_STRING + # ENDTODO + + + if connection_string.endswith(';'): + connection_string = connection_string[0:-1] + + connections_params = [entry.split('=')[1] for entry in connection_string.split(';')] + connection = MySQLdb.connect(*connections_params) + cursor = connection.cursor() + + logging.debug("mysql got query: %s", query) + + try: + cursor.execute(query) + + data = cursor.fetchall() + + num_fields = len(cursor.description) + column_names = [i[0] for i in cursor.description] + + for c in data: + logging.debug(c) + + rows = [dict(zip(column_names, row)) for row in data] + + + columns = [{'name': col_name, + 'friendly_name': column_friendly_name(col_name), + 'type': None} for col_name in column_names] + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + cursor.close() + except MySQLdb.Error, e: + json_data = None + error = e.message + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + finally: + connection.close() + + return json_data, error + + + return query_runner + + def redshift(connection_string): def column_friendly_name(column_name): From 1a6e5b425a056c63656bfcdc8487a6d44f55c784 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Wed, 29 Jan 2014 19:30:59 +0200 Subject: [PATCH 128/540] Include MySQL example --- rd_service/settings_example.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rd_service/settings_example.py b/rd_service/settings_example.py index af8c3d04e..d6ff69b51 100644 --- a/rd_service/settings_example.py +++ b/rd_service/settings_example.py @@ -5,7 +5,12 @@ Example settings module. You should make your own copy as settings.py and enter import django.conf REDIS_URL = "redis://localhost:6379" + +# Either "pg" or "mysql" +CONNECTION_ADAPTER = "mysql" # Connection string for the database that is used to run queries against +# -- example mysql CONNECTION_STRING = "Server=;User=;Pwd=;Database=" +# -- example pg CONNECTION_STRING = "user= password= host= port=5439 dbname=" CONNECTION_STRING = "user= password= host= port=5439 dbname=" # Connection string for the operational databases (where we store the queries, results, etc) INTERNAL_DB_CONNECTION_STRING = "dbname=postgres" From 95529ce8f09303fae9e267f3fbc6ab002c5e6c07 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Wed, 29 Jan 2014 20:57:09 +0200 Subject: [PATCH 129/540] Separated query runners to diff files --- rd_service/data/manager.py | 5 +- rd_service/data/query_runner.py | 53 ----------- rd_service/data/query_runner_mysql.py | 125 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 55 deletions(-) create mode 100644 rd_service/data/query_runner_mysql.py diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index 6854d85fc..27bcc2f42 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -9,7 +9,6 @@ import psycopg2 import qr import redis import time -import query_runner import worker import settings from utils import gen_query_hash @@ -156,8 +155,10 @@ class Manager(object): return self.workers if settings.CONNECTION_ADAPTER == "mysql": - runner = query_runner.mysql(connection_string) + import query_runner_mysql + runner = query_runner_mysql.mysql(connection_string) else: + import query_runner runner = query_runner.redshift(connection_string) self.workers = [worker.Worker(self, runner) for _ in range(workers_count)] diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index 0a23b52e5..c5248e547 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -8,7 +8,6 @@ query language (for example: HiveQL). """ import logging import json -import psycopg2 import MySQLdb import sys import select @@ -71,55 +70,3 @@ def mysql(connection_string): -def redshift(connection_string): - def column_friendly_name(column_name): - return column_name - - def wait(conn): - while 1: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) - - def query_runner(query): - connection = psycopg2.connect(connection_string, async=True) - wait(connection) - - cursor = connection.cursor() - - try: - cursor.execute(query) - wait(connection) - - column_names = [col.name for col in cursor.description] - - rows = [dict(zip(column_names, row)) for row in cursor] - columns = [{'name': col.name, - 'friendly_name': column_friendly_name(col.name), - 'type': None} for col in cursor.description] - - data = {'columns': columns, 'rows': rows} - json_data = json.dumps(data, cls=JSONEncoder) - error = None - cursor.close() - except psycopg2.DatabaseError as e: - json_data = None - error = e.message - except KeyboardInterrupt: - connection.cancel() - error = "Query cancelled by user." - json_data = None - except Exception as e: - raise sys.exc_info()[1], None, sys.exc_info()[2] - finally: - connection.close() - - return json_data, error - - return query_runner diff --git a/rd_service/data/query_runner_mysql.py b/rd_service/data/query_runner_mysql.py new file mode 100644 index 000000000..0a23b52e5 --- /dev/null +++ b/rd_service/data/query_runner_mysql.py @@ -0,0 +1,125 @@ +""" +QueryRunner is the function that the workers use, to execute queries. This is the Redshift +(PostgreSQL in fact) version, but easily we can write another to support additional databases +(MySQL and others). + +Because the worker just pass the query, this can be used with any data store that has some sort of +query language (for example: HiveQL). +""" +import logging +import json +import psycopg2 +import MySQLdb +import sys +import select +from .utils import JSONEncoder + +def mysql(connection_string): + def column_friendly_name(column_name): + return column_name + + def query_runner(query): + # TODO: remove these lines + import settings + connection_string = settings.CONNECTION_STRING + # ENDTODO + + + if connection_string.endswith(';'): + connection_string = connection_string[0:-1] + + connections_params = [entry.split('=')[1] for entry in connection_string.split(';')] + connection = MySQLdb.connect(*connections_params) + cursor = connection.cursor() + + logging.debug("mysql got query: %s", query) + + try: + cursor.execute(query) + + data = cursor.fetchall() + + num_fields = len(cursor.description) + column_names = [i[0] for i in cursor.description] + + for c in data: + logging.debug(c) + + rows = [dict(zip(column_names, row)) for row in data] + + + columns = [{'name': col_name, + 'friendly_name': column_friendly_name(col_name), + 'type': None} for col_name in column_names] + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + cursor.close() + except MySQLdb.Error, e: + json_data = None + error = e.message + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + finally: + connection.close() + + return json_data, error + + + return query_runner + + + +def redshift(connection_string): + def column_friendly_name(column_name): + return column_name + + def wait(conn): + while 1: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def query_runner(query): + connection = psycopg2.connect(connection_string, async=True) + wait(connection) + + cursor = connection.cursor() + + try: + cursor.execute(query) + wait(connection) + + column_names = [col.name for col in cursor.description] + + rows = [dict(zip(column_names, row)) for row in cursor] + columns = [{'name': col.name, + 'friendly_name': column_friendly_name(col.name), + 'type': None} for col in cursor.description] + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + cursor.close() + except psycopg2.DatabaseError as e: + json_data = None + error = e.message + except KeyboardInterrupt: + connection.cancel() + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + finally: + connection.close() + + return json_data, error + + return query_runner From a2385a1779ff9cfaa207acb8f7511356cc6f3d94 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Wed, 29 Jan 2014 21:02:12 +0200 Subject: [PATCH 130/540] Removed unecessary logging --- rd_service/data/query_runner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index c5248e547..cc0e8d7bb 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -41,12 +41,8 @@ def mysql(connection_string): num_fields = len(cursor.description) column_names = [i[0] for i in cursor.description] - for c in data: - logging.debug(c) - rows = [dict(zip(column_names, row)) for row in data] - columns = [{'name': col_name, 'friendly_name': column_friendly_name(col_name), 'type': None} for col_name in column_names] From fef5c287d74fe00fdeb72fc1bc6ca5b6d0a48c76 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Thu, 30 Jan 2014 11:28:11 +0200 Subject: [PATCH 131/540] Returned redshift code --- rd_service/data/query_runner.py | 73 +++++++++++++-------------- rd_service/data/query_runner_mysql.py | 1 - 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/rd_service/data/query_runner.py b/rd_service/data/query_runner.py index cc0e8d7bb..15b38a3ed 100644 --- a/rd_service/data/query_runner.py +++ b/rd_service/data/query_runner.py @@ -8,61 +8,60 @@ query language (for example: HiveQL). """ import logging import json -import MySQLdb +import psycopg2 import sys import select from .utils import JSONEncoder -def mysql(connection_string): +def redshift(connection_string): def column_friendly_name(column_name): return column_name - + + def wait(conn): + while 1: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + def query_runner(query): - # TODO: remove these lines - import settings - connection_string = settings.CONNECTION_STRING - # ENDTODO - - - if connection_string.endswith(';'): - connection_string = connection_string[0:-1] - - connections_params = [entry.split('=')[1] for entry in connection_string.split(';')] - connection = MySQLdb.connect(*connections_params) + connection = psycopg2.connect(connection_string, async=True) + wait(connection) + cursor = connection.cursor() - - logging.debug("mysql got query: %s", query) - + try: cursor.execute(query) - - data = cursor.fetchall() - - num_fields = len(cursor.description) - column_names = [i[0] for i in cursor.description] - - rows = [dict(zip(column_names, row)) for row in data] - - columns = [{'name': col_name, - 'friendly_name': column_friendly_name(col_name), - 'type': None} for col_name in column_names] - + wait(connection) + + column_names = [col.name for col in cursor.description] + + rows = [dict(zip(column_names, row)) for row in cursor] + columns = [{'name': col.name, + 'friendly_name': column_friendly_name(col.name), + 'type': None} for col in cursor.description] + data = {'columns': columns, 'rows': rows} json_data = json.dumps(data, cls=JSONEncoder) error = None cursor.close() - except MySQLdb.Error, e: + except psycopg2.DatabaseError as e: json_data = None error = e.message + except KeyboardInterrupt: + connection.cancel() + error = "Query cancelled by user." + json_data = None except Exception as e: raise sys.exc_info()[1], None, sys.exc_info()[2] finally: connection.close() - + return json_data, error - - - return query_runner - - - + + return query_runner \ No newline at end of file diff --git a/rd_service/data/query_runner_mysql.py b/rd_service/data/query_runner_mysql.py index 0a23b52e5..c85008167 100644 --- a/rd_service/data/query_runner_mysql.py +++ b/rd_service/data/query_runner_mysql.py @@ -8,7 +8,6 @@ query language (for example: HiveQL). """ import logging import json -import psycopg2 import MySQLdb import sys import select From b056e49ec52977b7bffa0fd29d17983a34f7429c Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Thu, 30 Jan 2014 11:28:49 +0200 Subject: [PATCH 132/540] Removed unnecessary logging --- rd_service/data/query_runner_mysql.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rd_service/data/query_runner_mysql.py b/rd_service/data/query_runner_mysql.py index c85008167..0ca6d5b3f 100644 --- a/rd_service/data/query_runner_mysql.py +++ b/rd_service/data/query_runner_mysql.py @@ -41,9 +41,6 @@ def mysql(connection_string): num_fields = len(cursor.description) column_names = [i[0] for i in cursor.description] - for c in data: - logging.debug(c) - rows = [dict(zip(column_names, row)) for row in data] From b0159c8246758bc2689c343ccb0e8aef2fbc4eb8 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Thu, 30 Jan 2014 16:03:58 +0200 Subject: [PATCH 133/540] Redshift shouldn't be here --- rd_service/data/query_runner_mysql.py | 67 +++------------------------ 1 file changed, 6 insertions(+), 61 deletions(-) diff --git a/rd_service/data/query_runner_mysql.py b/rd_service/data/query_runner_mysql.py index 0ca6d5b3f..2260b6a85 100644 --- a/rd_service/data/query_runner_mysql.py +++ b/rd_service/data/query_runner_mysql.py @@ -18,11 +18,11 @@ def mysql(connection_string): return column_name def query_runner(query): - # TODO: remove these lines - import settings - connection_string = settings.CONNECTION_STRING - # ENDTODO - + # # TODO: remove these lines + # import settings + # connection_string = settings.CONNECTION_STRING + # # ENDTODO + print connection_string if connection_string.endswith(';'): connection_string = connection_string[0:-1] @@ -63,59 +63,4 @@ def mysql(connection_string): return json_data, error - return query_runner - - - -def redshift(connection_string): - def column_friendly_name(column_name): - return column_name - - def wait(conn): - while 1: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) - - def query_runner(query): - connection = psycopg2.connect(connection_string, async=True) - wait(connection) - - cursor = connection.cursor() - - try: - cursor.execute(query) - wait(connection) - - column_names = [col.name for col in cursor.description] - - rows = [dict(zip(column_names, row)) for row in cursor] - columns = [{'name': col.name, - 'friendly_name': column_friendly_name(col.name), - 'type': None} for col in cursor.description] - - data = {'columns': columns, 'rows': rows} - json_data = json.dumps(data, cls=JSONEncoder) - error = None - cursor.close() - except psycopg2.DatabaseError as e: - json_data = None - error = e.message - except KeyboardInterrupt: - connection.cancel() - error = "Query cancelled by user." - json_data = None - except Exception as e: - raise sys.exc_info()[1], None, sys.exc_info()[2] - finally: - connection.close() - - return json_data, error - - return query_runner + return query_runner \ No newline at end of file From 6408b9e5e1617fb3d8779a14fda0473296373a70 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Thu, 30 Jan 2014 16:15:03 +0200 Subject: [PATCH 134/540] Fixed MySQL Runner --- rd_service/data/query_runner_mysql.py | 30 +++++++++------------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/rd_service/data/query_runner_mysql.py b/rd_service/data/query_runner_mysql.py index 2260b6a85..ff849598c 100644 --- a/rd_service/data/query_runner_mysql.py +++ b/rd_service/data/query_runner_mysql.py @@ -14,40 +14,30 @@ import select from .utils import JSONEncoder def mysql(connection_string): - def column_friendly_name(column_name): - return column_name - + if connection_string.endswith(';'): + connection_string = connection_string[0:-1] + def query_runner(query): - # # TODO: remove these lines - # import settings - # connection_string = settings.CONNECTION_STRING - # # ENDTODO - print connection_string - - if connection_string.endswith(';'): - connection_string = connection_string[0:-1] - connections_params = [entry.split('=')[1] for entry in connection_string.split(';')] connection = MySQLdb.connect(*connections_params) cursor = connection.cursor() - + logging.debug("mysql got query: %s", query) try: cursor.execute(query) data = cursor.fetchall() - + num_fields = len(cursor.description) column_names = [i[0] for i in cursor.description] rows = [dict(zip(column_names, row)) for row in data] - columns = [{'name': col_name, - 'friendly_name': column_friendly_name(col_name), + 'friendly_name': col_name, 'type': None} for col_name in column_names] - + data = {'columns': columns, 'rows': rows} json_data = json.dumps(data, cls=JSONEncoder) error = None @@ -59,8 +49,8 @@ def mysql(connection_string): raise sys.exc_info()[1], None, sys.exc_info()[2] finally: connection.close() - - return json_data, error - + return json_data, error + + return query_runner \ No newline at end of file From d2ace5c1cf80711299616411d1e8559e2593aefb Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 26 Jan 2014 15:06:10 +0200 Subject: [PATCH 135/540] Visualization UI: * queryfiddle page * new widget form --- rd_ui/app/scripts/controllers.js | 3 +- rd_ui/app/scripts/directives.js | 512 ++++++++++++++----------- rd_ui/app/scripts/services.js | 10 +- rd_ui/app/styles/redash.css | 4 + rd_ui/app/views/add_visualization.html | 36 ++ rd_ui/app/views/dashboard.html | 2 +- rd_ui/app/views/new_widget_form.html | 58 ++- rd_ui/app/views/queryfiddle.html | 11 + 8 files changed, 396 insertions(+), 240 deletions(-) create mode 100644 rd_ui/app/views/add_visualization.html diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 957f011cb..b00d54071 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -73,7 +73,8 @@ $scope.$parent.pageTitle = "Query Fiddle"; $scope.tabs = [{'key': 'table', 'name': 'Table'}, {'key': 'chart', 'name': 'Chart'}, - {'key': 'pivot', 'name': 'Pivot Table'}, {'key': 'cohort', 'name': 'Cohort'}]; + {'key': 'pivot', 'name': 'Pivot Table'}, {'key': 'cohort', 'name': 'Cohort'}, + {'key': 'add', 'name': 'Add Visualization'}]; $scope.lockButton = function (lock) { $scope.queryExecuting = lock; diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 92cb96060..ce5cafbd7 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -1,256 +1,310 @@ -var directives = angular.module('redash.directives', []); -directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) { - return { - restrict: 'E', - scope: { - tabsCollection: '=', - selectedTab: '=' - }, - template: '', - replace: true, - link: function($scope, element, attrs) { - $scope.selectTab = function(tabKey) { - $scope.selectedTab = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; }); - } +(function() { + 'use strict'; - $scope.$watch(function() { return $location.hash()}, function(hash) { - if (hash) { - $scope.selectTab($location.hash()); - } else { - $scope.selectTab($scope.tabsCollection[0].key); + var directives = angular.module('redash.directives', []); + + directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) { + return { + restrict: 'E', + scope: { + tabsCollection: '=', + selectedTab: '=' + }, + template: '', + replace: true, + link: function($scope, element, attrs) { + $scope.selectTab = function(tabKey) { + $scope.selectedTab = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; }); } - }); - } - } -}]) -directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) { - return { - restrict: 'E', - scope: { - dashboard: '=' - }, - templateUrl: '/views/edit_dashboard.html', - replace: true, - link: function($scope, element, attrs) { - var gridster = element.find(".gridster ul").gridster({ - widget_margins: [5, 5], - widget_base_dimensions: [260, 100], - min_cols: 2, - max_cols: 2, - serialize_params: function($w, wgd) { - return { - col: wgd.col, - row: wgd.row, - id: $w.data('widget-id') - } - } - }).data('gridster'); - - var gsItemTemplate = '
    • ' + - '
      {name}' + - '
      • '; - - $scope.$watch('dashboard.widgets', function(widgets) { - $timeout(function () { - gridster.remove_all_widgets(); - - if (widgets && widgets.length) { - var layout = []; - - _.each(widgets, function(row, rowIndex) { - _.each(row, function(widget, colIndex) { - layout.push({ - id: widget.id, - col: colIndex+1, - row: rowIndex+1, - ySize: 1, - xSize: widget.width, - name: widget.query.name - }); - }); - }); - - _.each(layout, function(item) { - var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); - gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); - - }); + $scope.$watch(function() { return $location.hash()}, function(hash) { + if (hash) { + $scope.selectTab($location.hash()); + } else { + $scope.selectTab($scope.tabsCollection[0].key); } }); - }, true); - - $scope.saveDashboard = function() { - $scope.saveInProgress = true; - // TODO: we should use the dashboard service here. - if ($scope.dashboard.id) { - var positions = $(element).find('.gridster ul').data('gridster').serialize(); - var layout = []; - _.each(_.sortBy(positions, function (pos) { - return pos.row * 10 + pos.col; - }), function (pos) { - var row = pos.row - 1; - var col = pos.col - 1; - layout[row] = layout[row] || []; - if (col > 0 && layout[row][col - 1] == undefined) { - layout[row][col - 1] = pos.id; - } else { - layout[row][col] = pos.id; - } - - }); - $scope.dashboard.layout = layout; - - layout = JSON.stringify(layout); - $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) { - $scope.dashboard = new Dashboard(response); - $scope.saveInProgress = false; - $(element).modal('hide'); - }) - } else { - $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) { - $(element).modal('hide'); - $location.path('/dashboard/' + response.slug).replace(); - }) - } } - } - } -}]) + }]); + directives.directive('addVisulatizationForm', [function() { + return { + restrict: 'E', + templateUrl: '/views/add_visualization.html', + replace: true, + scope: {}, + link: function(scope) { + scope.visTypes = [ + {type: 'pivot', name: 'Pivot Table'}, + {type: 'cohort', name: 'Cohort'} + ]; -directives.directive('newWidgetForm', ['$http', function($http) { - return { - restrict: 'E', - scope: { - dashboard: '=' - }, - templateUrl: '/views/new_widget_form.html', - replace: true, - link: function($scope, element, attrs) { - $scope.widgetTypes = [{name: 'Chart', value: 'chart'}, {name: 'Table', value: 'grid'}, {name: 'Cohort', value: 'cohort'}]; - $scope.widgetSizes = [{name: 'Regular Size', value: 1}, {name: 'Double Size', value: 2}]; + scope.vis = { + type: 'pivot', + name: null, + config: null + }; - var reset = function() { - $scope.saveInProgress = false; - $scope.widgetType = 'chart'; - $scope.widgetSize = 1; - $scope.queryId = null; + scope.typeChanged = function() { + console.log('evme', 'typeChanged'); + scope.vis.config = {}; + }; + + scope.submit = function() { + alert('Not implemented'); + }; } + } + }]); - reset(); + directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) { + return { + restrict: 'E', + scope: { + dashboard: '=' + }, + templateUrl: '/views/edit_dashboard.html', + replace: true, + link: function($scope, element, attrs) { + var gridster = element.find(".gridster ul").gridster({ + widget_margins: [5, 5], + widget_base_dimensions: [260, 100], + min_cols: 2, + max_cols: 2, + serialize_params: function($w, wgd) { + return { + col: wgd.col, + row: wgd.row, + id: $w.data('widget-id') + } + } + }).data('gridster'); - $scope.saveWidget = function() { - $scope.saveInProgress = true; + var gsItemTemplate = '
    • ' + + '
      {name}' + + '
      • '; - var widget = { - 'query_id': $scope.queryId, - 'dashboard_id': $scope.dashboard.id, - 'type': $scope.widgetType, - 'options': {}, - 'width': $scope.widgetSize + $scope.$watch('dashboard.widgets', function(widgets) { + $timeout(function () { + gridster.remove_all_widgets(); + + if (widgets && widgets.length) { + var layout = []; + + _.each(widgets, function(row, rowIndex) { + _.each(row, function(widget, colIndex) { + layout.push({ + id: widget.id, + col: colIndex+1, + row: rowIndex+1, + ySize: 1, + xSize: widget.width, + name: widget.query.name + }); + }); + }); + + _.each(layout, function(item) { + var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); + gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); + + }); + } + }); + }, true); + + $scope.saveDashboard = function() { + $scope.saveInProgress = true; + // TODO: we should use the dashboard service here. + if ($scope.dashboard.id) { + var positions = $(element).find('.gridster ul').data('gridster').serialize(); + var layout = []; + _.each(_.sortBy(positions, function (pos) { + return pos.row * 10 + pos.col; + }), function (pos) { + var row = pos.row - 1; + var col = pos.col - 1; + layout[row] = layout[row] || []; + if (col > 0 && layout[row][col - 1] == undefined) { + layout[row][col - 1] = pos.id; + } else { + layout[row][col] = pos.id; + } + + }); + $scope.dashboard.layout = layout; + + layout = JSON.stringify(layout); + $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) { + $scope.dashboard = new Dashboard(response); + $scope.saveInProgress = false; + $(element).modal('hide'); + }) + } else { + $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) { + $(element).modal('hide'); + $location.path('/dashboard/' + response.slug).replace(); + }) + } } - $http.post('/api/widgets', widget).success(function(response) { - // update dashboard layout - $scope.dashboard.layout = response['layout']; - if (response['new_row']) { - $scope.dashboard.widgets.push([response['widget']]); - } else { - $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']); + } + } + }]); + + directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) { + return { + restrict: 'E', + scope: { + dashboard: '=' + }, + templateUrl: '/views/new_widget_form.html', + replace: true, + link: function($scope, element, attrs) { + $scope.visReady = false; + $scope.currentView = 'existing'; + $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}]; + + var reset = function() { + $scope.saveInProgress = false; + $scope.widgetSize = 1; + $scope.queryId = null; + } + + reset(); + + $scope.toggleView = function(viewName) { + $scope.currentView = ($scope.currentView == viewName) ? '' : viewName; + }; + + $scope.loadVisualizations = function() { + Query.get({ + id: $scope.queryId + }, function(query) { + if (query) { + // TODO + $scope.visReady = true; + var visualizations = query.getVisualizations(); + $scope.existing = visualizations = [ + {'name': 'vis1', 'type': 'Cohort'}, + {'name': 'vis2', 'type': 'Pivot Table'} + ]; + $scope.currentView = (visualizations.length) ? 'existing' : 'addNew'; + } + }); + }; + + $scope.saveWidget = function() { + $scope.saveInProgress = true; + + var widget = { + 'query_id': $scope.queryId, + 'dashboard_id': $scope.dashboard.id, + 'options': {}, + 'width': $scope.widgetSize } - // close the dialog - $('#add_query_dialog').modal('hide'); - reset(); + $http.post('/api/widgets', widget).success(function(response) { + // update dashboard layout + $scope.dashboard.layout = response['layout']; + if (response['new_row']) { + $scope.dashboard.widgets.push([response['widget']]); + } else { + $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']); + } + + // close the dialog + $('#add_query_dialog').modal('hide'); + reset(); + }) + } + + } + } + }]) + + // From: http://jsfiddle.net/joshdmiller/NDFHg/ + directives.directive('editInPlace', function () { + return { + restrict: 'E', + scope: { + value: '=', + ignoreBlanks: '=', + editable: '=' + }, + template: function(tElement, tAttrs) { + var elType = tAttrs.editor || 'input'; + var placeholder = tAttrs.placeholder || 'Click to edit'; + return '' + + '' + placeholder + '' + + '<{elType} ng-model="value" class="form-control" rows="2">'.replace('{elType}', elType); + }, + link: function ($scope, element, attrs) { + // Let's get a reference to the input element, as we'll want to reference it. + var inputElement = angular.element(element.children()[2]); + + // This directive should have a set class so we can style it. + element.addClass('edit-in-place'); + + // Initially, we're not editing. + $scope.editing = false; + + // ng-click handler to activate edit-in-place + $scope.edit = function () { + if ($scope.ignoreBlanks) { + $scope.oldValue = $scope.value; + } + + $scope.editing = true; + + // We control display through a class on the directive itself. See the CSS. + element.addClass('active'); + + // And we must focus the element. + // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function, + // we have to reference the first element in the array. + inputElement[0].focus(); + }; + + $(inputElement).blur(function() { + if ($scope.ignoreBlanks && _.isEmpty($scope.value)) { + $scope.value = $scope.oldValue; + } + $scope.editing = false; + element.removeClass('active'); }) } + }; + }); - } - } -}]) + directives.directive('rdTimer', ['$timeout', function ($timeout) { + return { + restrict: 'E', + scope: { timestamp: '=' }, + template: '{{currentTime}}', + controller: ['$scope' ,function ($scope) { + $scope.currentTime = "00:00:00"; + var currentTimeout = null; -// From: http://jsfiddle.net/joshdmiller/NDFHg/ -directives.directive('editInPlace', function () { - return { - restrict: 'E', - scope: { - value: '=', - ignoreBlanks: '=', - editable: '=' - }, - template: function(tElement, tAttrs) { - var elType = tAttrs.editor || 'input'; - var placeholder = tAttrs.placeholder || 'Click to edit'; - return '' + - '' + placeholder + '' + - '<{elType} ng-model="value" class="form-control" rows="2">'.replace('{elType}', elType); - }, - link: function ($scope, element, attrs) { - // Let's get a reference to the input element, as we'll want to reference it. - var inputElement = angular.element(element.children()[2]); - - // This directive should have a set class so we can style it. - element.addClass('edit-in-place'); - - // Initially, we're not editing. - $scope.editing = false; - - // ng-click handler to activate edit-in-place - $scope.edit = function () { - if ($scope.ignoreBlanks) { - $scope.oldValue = $scope.value; + var updateTime = function() { + $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss") + currentTimeout = $timeout(updateTime, 1000); } - $scope.editing = true; - - // We control display through a class on the directive itself. See the CSS. - element.addClass('active'); - - // And we must focus the element. - // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function, - // we have to reference the first element in the array. - inputElement[0].focus(); - }; - - $(inputElement).blur(function() { - if ($scope.ignoreBlanks && _.isEmpty($scope.value)) { - $scope.value = $scope.oldValue; + var cancelTimer = function() { + if (currentTimeout) { + $timeout.cancel(currentTimeout); + currentTimeout = null; + } } - $scope.editing = false; - element.removeClass('active'); - }) - } - }; -}); -directives.directive('rdTimer', ['$timeout', function ($timeout) { - return { - restrict: 'E', - scope: { timestamp: '=' }, - template: '{{currentTime}}', - controller: ['$scope' ,function ($scope) { - $scope.currentTime = "00:00:00"; - var currentTimeout = null; + updateTime(); - var updateTime = function() { - $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss") - currentTimeout = $timeout(updateTime, 1000); - } - - var cancelTimer = function() { - if (currentTimeout) { - $timeout.cancel(currentTimeout); - currentTimeout = null; - } - } - - updateTime(); - - $scope.$on('$destroy', function () { - cancelTimer(); - }); - }] - }; -}]); \ No newline at end of file + $scope.$on('$destroy', function () { + cancelTimer(); + }); + }] + }; + }]); +})(); diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 230d2d7e1..f0da81acd 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -273,11 +273,17 @@ } return queryResult; - } + }; Query.prototype.getHash = function() { return [this.name, this.description, this.query].join('!#'); - } + }; + + Query.prototype.getVisualizations = function() { + // TODO + // not implemented + return []; + }; return Query; } diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 843ccff76..b5301a523 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -56,6 +56,10 @@ a.navbar-brand { margin-bottom: 0px; } +.panel-heading > a { + color: inherit; +} + /* angular-growl */ .growl { position: fixed; diff --git a/rd_ui/app/views/add_visualization.html b/rd_ui/app/views/add_visualization.html new file mode 100644 index 000000000..5de8b6ea5 --- /dev/null +++ b/rd_ui/app/views/add_visualization.html @@ -0,0 +1,36 @@ +
      +
      + + +
      + +
      + + +
      + + +
      + +
      +
      + +
      + + +
      + +
      +
      + +
      + + +
      + +
      + +
      + {{vis}} +
      +
      diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 6892233c1..1fa941d30 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -8,7 +8,7 @@ diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 01455d4dd..50a52e297 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -7,15 +7,59 @@

      - +

      +
      + +
      + +

      -

      - -

      -

      - -

      +
      +
      + +
      + +
      +
      + + Existing + +
      +
      +
      +
        +
      • + {{item.name}} {{item.type}} +
        • +
      +
      +
      +
      + + +
      +
      + + Add New + +
      +
      +
      + +
      +
      +
      +
      +
      + +
      + + +
      +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index bcdcec8d4..812d81221 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -68,5 +68,16 @@
      + +
      +
      +
      +

      Add New Visualization

      +
      +
      + +
      +
      +
      \ No newline at end of file From cab6f9e58d8d2782619061aa5bdb5cede61f42b4 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 27 Jan 2014 17:37:41 +0200 Subject: [PATCH 136/540] Visualization models --- rd_service/data/models.py | 30 ++++++++++++++++++++++++++++-- rd_service/data/tables.sql | 11 ++++++++++- rd_service/server.py | 11 ++++++++++- rd_ui/app/scripts/services.js | 10 ++++++++-- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index 58c9e9850..f1b613622 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -148,10 +148,36 @@ class Dashboard(models.Model): return u"%s=%s" % (self.id, self.name) +class Visualization(models.Model): + id = models.AutoField(primary_key=True) + type = models.CharField(max_length=100) + query = models.ForeignKey(Query, related_name='visualizations') + name = models.CharField(max_length=100) + description = models.CharField(max_length=4096) + options = models.TextField() + + class Meta: + app_label = 'redash' + db_table = 'visualizations' + + def to_dict(self): + return { + 'id': self.id, + 'query': self.query.to_dict(), + 'type': self.type, + 'name': self.name, + 'description': self.description, + 'options': json.loads(self.options), + } + + def __unicode__(self): + return u"%s=>%s" % (self.id, self.query_id) + + class Widget(models.Model): id = models.AutoField(primary_key=True) - query = models.ForeignKey(Query) type = models.CharField(max_length=100) + visualization = models.ForeignKey(Visualization, related_name='widgets') width = models.IntegerField() options = models.TextField() dashboard = models.ForeignKey(Dashboard, related_name='widgets') @@ -163,10 +189,10 @@ class Widget(models.Model): def to_dict(self): return { 'id': self.id, - 'query': self.query.to_dict(), 'type': self.type, 'width': self.width, 'options': json.loads(self.options), + 'visualization_id': self.visualization_id, 'dashboard_id': self.dashboard_id } diff --git a/rd_service/data/tables.sql b/rd_service/data/tables.sql index fa6b941ab..4e1f24ded 100644 --- a/rd_service/data/tables.sql +++ b/rd_service/data/tables.sql @@ -30,12 +30,21 @@ CREATE TABLE "dashboards" ( "is_archived" boolean NOT NULL ) ; +CREATE TABLE "visualizations" ( + "id" serial NOT NULL PRIMARY KEY, + "type" varchar(100) NOT NULL, + "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, + "name" varchar(100) NOT NULL, + "description" varchar(4096), + "options" text NOT NULL +) +; CREATE TABLE "widgets" ( "id" serial NOT NULL PRIMARY KEY, - "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, "type" varchar(100) NOT NULL, "width" integer NOT NULL, "options" text NOT NULL, + "visualization_id" integer NOT NULL REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED, "dashboard_id" integer NOT NULL REFERENCES "dashboards" ("id") DEFERRABLE INITIALLY DEFERRED ) ; diff --git a/rd_service/server.py b/rd_service/server.py index 306fbfa08..712d29205 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -167,7 +167,7 @@ class WidgetsHandler(BaseAuthenticatedHandler): class DashboardHandler(BaseAuthenticatedHandler): def get(self, dashboard_slug=None): if dashboard_slug: - dashboard = data.models.Dashboard.objects.prefetch_related('widgets__query__latest_query_data').get(slug=dashboard_slug) + dashboard = data.models.Dashboard.objects.prefetch_related('widgets__visualization__query__latest_query_data').get(slug=dashboard_slug) self.write_json(dashboard.to_dict(with_widgets=True)) else: dashboards = [d.to_dict() for d in @@ -251,6 +251,14 @@ class QueryResultsHandler(BaseAuthenticatedHandler): self.write({'job': job.to_dict()}) +class VisualizationHandler(BaseAuthenticatedHandler): + def get(self, id): + pass + + def post(self, id=None): + pass + + class CsvQueryResultsHandler(BaseAuthenticatedHandler): def get_current_user(self): user = super(CsvQueryResultsHandler, self).get_current_user() @@ -312,6 +320,7 @@ def get_application(static_path, is_debug, redis_connection, data_manager): (r"/api/queries(?:/([0-9]*))?", QueriesHandler), (r"/api/query_results(?:/([0-9]*))?", QueryResultsHandler), (r"/api/jobs/(.*)", JobsHandler), + (r"/api/visualizations(?:/([0-9]*))?", VisualizationHandler), (r"/api/widgets(?:/([0-9]*))?", WidgetsHandler), (r"/api/dashboards(?:/(.*))?", DashboardHandler), (r"/admin/(.*)", MainHandler), diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index f0da81acd..90907726e 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -253,7 +253,7 @@ } return QueryResult; - } + }; var Query = function ($resource, QueryResult) { var Query = $resource('/api/queries/:id', {id: '@id'}); @@ -286,10 +286,16 @@ }; return Query; - } + }; + + var Visualization = function($resource) { + var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); + return Visualization; + }; angular.module('redash.services', []) .factory('QueryResult', ['$resource', '$timeout', QueryResult]) .factory('Query', ['$resource', 'QueryResult', Query]) + .factory('Visualization', ['$resource', 'Visualization', Visualization]) })(); From 84df2fb85c3f31420982bea30aa19ad3619ae964 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 28 Jan 2014 16:51:19 +0200 Subject: [PATCH 137/540] create Visualization [WIP] --- rd_service/server.py | 17 +++++++++- rd_ui/app/scripts/directives.js | 27 +++++++++------- rd_ui/app/scripts/services.js | 17 +++++++++- rd_ui/app/views/add_visualization.html | 45 +++++++++++++++++--------- rd_ui/app/views/queryfiddle.html | 2 +- 5 files changed, 77 insertions(+), 31 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index 712d29205..4d9ab016c 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -256,7 +256,22 @@ class VisualizationHandler(BaseAuthenticatedHandler): pass def post(self, id=None): - pass + kwargs = json.loads(self.request.body) + + if id: + vis = data.models.Visualization(**kwargs) + fields = kwargs.keys() + fields.remove('id') + vis.save(update_fields=fields) + else: + query_id = kwargs.pop('query_id', None) + query = data.models.Query.objects.get(pk=query_id) if query_id else None + kwargs['query'] = query + + vis = data.models.Visualization(**kwargs) + vis.save() + + self.write_json(vis.to_dict(with_result=False)) class CsvQueryResultsHandler(BaseAuthenticatedHandler): diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index ce5cafbd7..24dc2e94d 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -28,31 +28,34 @@ } }]); - directives.directive('addVisulatizationForm', [function() { + directives.directive('addVisulatizationForm', ['Visualization', function(Visualization) { return { restrict: 'E', templateUrl: '/views/add_visualization.html', replace: true, - scope: {}, + scope: { + queryId: "=" + }, link: function(scope) { - scope.visTypes = [ - {type: 'pivot', name: 'Pivot Table'}, - {type: 'cohort', name: 'Cohort'} - ]; + scope.visTypes = _.map(Visualization.prototype.TYPES, function (type) { + return {type: type, name: Visualization.prototype.NAMES[type]}; + }); scope.vis = { - type: 'pivot', - name: null, - config: null + 'query_id': scope.queryId, + 'type': Visualization.prototype.TYPES.PIVOT, + 'name': '', + 'description': '', + 'options': null }; scope.typeChanged = function() { console.log('evme', 'typeChanged'); - scope.vis.config = {}; + scope.vis.options = {}; }; - scope.submit = function() { - alert('Not implemented'); + scope.createVis = function() { + Visualization.save(scope.vis); }; } } diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 90907726e..19dc3925b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -289,13 +289,28 @@ }; var Visualization = function($resource) { + var PIVOT = 'PIVOT'; + var COHORT = 'COHORT'; + var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); + + Visualization.prototype = { + TYPES: { + 'PIVOT': PIVOT, + 'COHORT': COHORT + }, + NAMES: { + 'PIVOT': 'Pivot Table', + 'COHORT': 'Cohort' + } + }; + return Visualization; }; angular.module('redash.services', []) .factory('QueryResult', ['$resource', '$timeout', QueryResult]) .factory('Query', ['$resource', 'QueryResult', Query]) - .factory('Visualization', ['$resource', 'Visualization', Visualization]) + .factory('Visualization', ['$resource', Visualization]) })(); diff --git a/rd_ui/app/views/add_visualization.html b/rd_ui/app/views/add_visualization.html index 5de8b6ea5..dfb2548ce 100644 --- a/rd_ui/app/views/add_visualization.html +++ b/rd_ui/app/views/add_visualization.html @@ -1,33 +1,46 @@ -
      +
      +
      + + +
      +
      - -
      - -
      -
      - -
      +
      + - -
      - -
      -
      - -
      + +
      +

      + +

      +

      + +

      +
      + +
      +

      + +

      +

      + +

      +
      + +
      - +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 812d81221..b7ef29d5a 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -75,7 +75,7 @@

      Add New Visualization

      - +
      From 8327baa2f677191cfaa22ad54b10213e87ea9b46 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 29 Jan 2014 17:41:49 +0200 Subject: [PATCH 138/540] create Visualization cont. --- rd_service/data/models.py | 17 +++++++++++++---- rd_service/server.py | 10 ++++------ rd_ui/app/scripts/controllers.js | 13 ++++++++++--- rd_ui/app/scripts/directives.js | 23 ++++++++++++++--------- rd_ui/app/scripts/services.js | 7 ++++--- rd_ui/app/views/add_visualization.html | 18 +++++++++--------- rd_ui/app/views/queryfiddle.html | 24 ++++++++++++------------ 7 files changed, 66 insertions(+), 46 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index f1b613622..7ca0abfb1 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -51,7 +51,8 @@ class Query(models.Model): app_label = 'redash' db_table = 'queries' - def to_dict(self, with_result=True, with_stats=False): + def to_dict(self, with_result=True, with_stats=False, + with_visualizations=False): d = { 'id': self.id, 'latest_query_data_id': self.latest_query_data_id, @@ -75,6 +76,10 @@ class Query(models.Model): if with_result and self.latest_query_data_id: d['latest_query_data'] = self.latest_query_data.to_dict() + if with_visualizations: + d['visualizations'] = [vis.to_dict(with_query=False) + for vis in self.visualizations.all()] + return d @classmethod @@ -160,16 +165,20 @@ class Visualization(models.Model): app_label = 'redash' db_table = 'visualizations' - def to_dict(self): - return { + def to_dict(self, with_query=True): + d = { 'id': self.id, - 'query': self.query.to_dict(), 'type': self.type, 'name': self.name, 'description': self.description, 'options': json.loads(self.options), } + if with_query: + d['query'] = self.query.to_dict() + + return d + def __unicode__(self): return u"%s=>%s" % (self.id, self.query_id) diff --git a/rd_service/server.py b/rd_service/server.py index 4d9ab016c..75fcbf16b 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -204,6 +204,7 @@ class QueriesHandler(BaseAuthenticatedHandler): query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) query_def.pop('latest_query_data', None) + query_def.pop('visualizations', None) if id: query = data.models.Query(**query_def) @@ -221,7 +222,7 @@ class QueriesHandler(BaseAuthenticatedHandler): if id: q = data.models.Query.objects.get(pk=id) if q: - self.write_json(q.to_dict()) + self.write_json(q.to_dict(with_visualizations=True)) else: self.send_error(404) else: @@ -257,6 +258,7 @@ class VisualizationHandler(BaseAuthenticatedHandler): def post(self, id=None): kwargs = json.loads(self.request.body) + kwargs['options'] = json.dumps(kwargs['options']) if id: vis = data.models.Visualization(**kwargs) @@ -264,14 +266,10 @@ class VisualizationHandler(BaseAuthenticatedHandler): fields.remove('id') vis.save(update_fields=fields) else: - query_id = kwargs.pop('query_id', None) - query = data.models.Query.objects.get(pk=query_id) if query_id else None - kwargs['query'] = query - vis = data.models.Visualization(**kwargs) vis.save() - self.write_json(vis.to_dict(with_result=False)) + self.write_json(vis.to_dict()) class CsvQueryResultsHandler(BaseAuthenticatedHandler): diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index b00d54071..24540519c 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -72,9 +72,11 @@ $scope.$parent.pageTitle = "Query Fiddle"; - $scope.tabs = [{'key': 'table', 'name': 'Table'}, {'key': 'chart', 'name': 'Chart'}, - {'key': 'pivot', 'name': 'Pivot Table'}, {'key': 'cohort', 'name': 'Cohort'}, - {'key': 'add', 'name': 'Add Visualization'}]; + // more tabs added when query loads + $scope.tabs = [{'key': 'table', 'name': 'Table'}, + {'key': 'pivot', 'name': 'Pivot Table'}, + {'key': 'add', 'name': 'New Visualization'}]; + $scope.lockButton = function (lock) { $scope.queryExecuting = lock; @@ -192,6 +194,11 @@ pristineHash = q.getHash(); $scope.dirty = false; $scope.queryResult = $scope.query.getQueryResult(); + + _.each(q.visualizations, function(vis) { + $scope.tabs.splice(-1, 0, {'key': 'vis' + vis.id, 'name': vis.name}); + }); + }); } else { $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser.name}); diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 24dc2e94d..ebe99bdc7 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -28,25 +28,26 @@ } }]); - directives.directive('addVisulatizationForm', ['Visualization', function(Visualization) { + directives.directive('addVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { return { restrict: 'E', templateUrl: '/views/add_visualization.html', replace: true, scope: { - queryId: "=" + query: "=" }, link: function(scope) { - scope.visTypes = _.map(Visualization.prototype.TYPES, function (type) { + scope.visTypes = Visualization.prototype.TYPES; + scope.visOptions = _.map(Visualization.prototype.TYPES, function (type) { return {type: type, name: Visualization.prototype.NAMES[type]}; }); scope.vis = { - 'query_id': scope.queryId, - 'type': Visualization.prototype.TYPES.PIVOT, - 'name': '', - 'description': '', - 'options': null + 'query_id': scope.query.id, + 'type': scope.visTypes.CHART, + 'name': scope.query.name, + 'description': scope.query.description, + 'options': {} }; scope.typeChanged = function() { @@ -55,7 +56,11 @@ }; scope.createVis = function() { - Visualization.save(scope.vis); + Visualization.save(scope.vis, function success(result) { + scope.vis = result; + }, function error() { + growl.addErrorMessage("Visualization could not be saved"); + }); }; } } diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 19dc3925b..919804208 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -263,6 +263,7 @@ ttl = this.ttl; } + var queryResult = null; if (this.latest_query_data && ttl != 0) { queryResult = new QueryResult({'query_result': this.latest_query_data}); @@ -289,18 +290,18 @@ }; var Visualization = function($resource) { - var PIVOT = 'PIVOT'; + var CHART = 'CHART'; var COHORT = 'COHORT'; var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); Visualization.prototype = { TYPES: { - 'PIVOT': PIVOT, + 'CHART': CHART, 'COHORT': COHORT }, NAMES: { - 'PIVOT': 'Pivot Table', + 'CHART': 'Chart', 'COHORT': 'Cohort' } }; diff --git a/rd_ui/app/views/add_visualization.html b/rd_ui/app/views/add_visualization.html index dfb2548ce..5c0c27ab9 100644 --- a/rd_ui/app/views/add_visualization.html +++ b/rd_ui/app/views/add_visualization.html @@ -1,34 +1,34 @@
      - +
      - +
      - +
      - -
      + +

      - +

      - +

      -
      +

      @@ -40,7 +40,7 @@
      - +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index b7ef29d5a..6496eaf2f 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -53,9 +53,6 @@
      -
      - -
      @@ -65,18 +62,21 @@
      -
      - +
      +

      {{vis}}

      -
      -
      -

      Add New Visualization

      -
      -
      - -
      +
      +

      +

      + +
      +
      + + +
      +

      From fdd2cfe1d147281bee35bfc639efad9bf57bb06c Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 30 Jan 2014 17:33:02 +0200 Subject: [PATCH 139/540] edit and create visualizations --- rd_service/server.py | 2 +- rd_ui/app/scripts/controllers.js | 23 +++---- rd_ui/app/scripts/directives.js | 61 +++++++++++++++---- ...alization.html => edit_visualization.html} | 2 +- rd_ui/app/views/queryfiddle.html | 38 ++++++++---- 5 files changed, 86 insertions(+), 40 deletions(-) rename rd_ui/app/views/{add_visualization.html => edit_visualization.html} (96%) diff --git a/rd_service/server.py b/rd_service/server.py index 75fcbf16b..dbeef29ee 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -269,7 +269,7 @@ class VisualizationHandler(BaseAuthenticatedHandler): vis = data.models.Visualization(**kwargs) vis.save() - self.write_json(vis.to_dict()) + self.write_json(vis.to_dict(with_query=False)) class CsvQueryResultsHandler(BaseAuthenticatedHandler): diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 24540519c..725e0bd2a 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -33,12 +33,14 @@ $scope.updateTime = ''; } - var QueryFiddleCtrl = function ($scope, $window, $routeParams, $http, $location, growl, notifications, Query) { + var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query) { + var DEFAULT_TAB = 'table'; var pristineHash = null; - $scope.dirty = undefined; - var leavingPageText = "You will lose your changes if you leave"; + $scope.dirty = undefined; + $scope.newVisualization = undefined; + $window.onbeforeunload = function(){ if (currentUser.canEdit($scope.query) && $scope.dirty) { return leavingPageText; @@ -72,11 +74,9 @@ $scope.$parent.pageTitle = "Query Fiddle"; - // more tabs added when query loads - $scope.tabs = [{'key': 'table', 'name': 'Table'}, - {'key': 'pivot', 'name': 'Pivot Table'}, - {'key': 'add', 'name': 'New Visualization'}]; - + $scope.$watch(function() {return $location.hash()}, function(hash) { + $scope.selectedTab = hash || DEFAULT_TAB; + }); $scope.lockButton = function (lock) { $scope.queryExecuting = lock; @@ -194,11 +194,6 @@ pristineHash = q.getHash(); $scope.dirty = false; $scope.queryResult = $scope.query.getQueryResult(); - - _.each(q.visualizations, function(vis) { - $scope.tabs.splice(-1, 0, {'key': 'vis' + vis.id, 'name': vis.name}); - }); - }); } else { $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser.name}); @@ -381,7 +376,7 @@ .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('QueryFiddleCtrl', ['$scope', '$window', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) + .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index ebe99bdc7..ca0a955ff 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -3,6 +3,23 @@ var directives = angular.module('redash.directives', []); + directives.directive('rdTab', ['$location', function($location) { + return { + restrict: 'E', + scope: { + 'id': '@', + 'name': '@' + }, + template: '
    • {{name}}
      • ', + replace: true, + link: function(scope) { + scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) { + scope.selectedTab = tab; + }); + } + } + }]); + directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) { return { restrict: 'E', @@ -28,36 +45,56 @@ } }]); - directives.directive('addVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { + directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { return { restrict: 'E', - templateUrl: '/views/add_visualization.html', + templateUrl: '/views/edit_visualization.html', replace: true, scope: { - query: "=" + query: '=', + vis: '=?' }, - link: function(scope) { + link: function(scope, element, attrs) { scope.visTypes = Visualization.prototype.TYPES; scope.visOptions = _.map(Visualization.prototype.TYPES, function (type) { return {type: type, name: Visualization.prototype.NAMES[type]}; }); - scope.vis = { - 'query_id': scope.query.id, - 'type': scope.visTypes.CHART, - 'name': scope.query.name, - 'description': scope.query.description, - 'options': {} - }; + if (!scope.vis) { + // create new visualization + // wait for query to load to populate with defaults + var unwatch = scope.$watch('query', function(q) { + if (q.id) { + unwatch(); + scope.vis = { + 'query_id': q.id, + 'type': scope.visTypes.CHART, + 'name': q.name, + 'description': q.description, + 'options': {} + }; + } + }, true); + } scope.typeChanged = function() { console.log('evme', 'typeChanged'); scope.vis.options = {}; }; - scope.createVis = function() { + scope.submit = function() { Visualization.save(scope.vis, function success(result) { + growl.addSuccessMessage("Visualization saved"); + scope.vis = result; + + var visIds = _.pluck(scope.query.visualizations, 'id'); + var index = visIds.indexOf(result.id); + if (index > -1) { + scope.query.visualizations[index] = result; + } else { + scope.query.visualizations.push(result); + } }, function error() { growl.addErrorMessage("Visualization could not be saved"); }); diff --git a/rd_ui/app/views/add_visualization.html b/rd_ui/app/views/edit_visualization.html similarity index 96% rename from rd_ui/app/views/add_visualization.html rename to rd_ui/app/views/edit_visualization.html index 5c0c27ab9..3c85498df 100644 --- a/rd_ui/app/views/add_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -1,4 +1,4 @@ - +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 6496eaf2f..76dc905e6 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -51,30 +51,44 @@
      - +
        + + + + +
      - -
      +
      -
      +
      -
      -

      {{vis}}

      -
      - -
      +

      - +
      +
      + + +
      +

      +
      +
      + +
      +
      +

      - - + +
      +
      + +

      From c83705119d93d95b5f9860f404a34627ddbebb6a Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 2 Feb 2014 13:16:20 +0200 Subject: [PATCH 140/540] delete visualizations --- rd_service/server.py | 4 ++++ rd_ui/app/scripts/controllers.js | 21 +++++++++++++++++---- rd_ui/app/scripts/directives.js | 3 ++- rd_ui/app/styles/redash.css | 11 +++++++++++ rd_ui/app/views/queryfiddle.html | 6 ++++-- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/rd_service/server.py b/rd_service/server.py index dbeef29ee..5b09079eb 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -271,6 +271,10 @@ class VisualizationHandler(BaseAuthenticatedHandler): self.write_json(vis.to_dict(with_query=False)) + def delete(self, id): + vis = data.models.Visualization.objects.get(pk=id) + vis.delete() + class CsvQueryResultsHandler(BaseAuthenticatedHandler): def get_current_user(self): diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 725e0bd2a..f7c7ff979 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -33,7 +33,7 @@ $scope.updateTime = ''; } - var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query) { + var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) { var DEFAULT_TAB = 'table'; var pristineHash = null; var leavingPageText = "You will lose your changes if you leave"; @@ -214,13 +214,26 @@ $scope.queryResult = $scope.query.getQueryResult(0); $scope.lockButton(true); $scope.cancelling = false; - } + }; $scope.cancelExecution = function() { $scope.cancelling = true; $scope.queryResult.cancelExecution(); - } + }; + $scope.deleteVisualization = function($e, vis) { + $e.preventDefault(); + if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { + Visualization.delete(vis); + if ($scope.selectedTab == vis.id) { + $scope.selectedTab = DEFAULT_TAB; + } + $scope.query.visualizations = + $scope.query.visualizations.filter(function(v) { + return vis.id !== v.id; + }); + } + }; } var QueriesCtrl = function($scope, $http, $location, $filter, Query) { @@ -376,7 +389,7 @@ .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', QueryFiddleCtrl]) + .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index ca0a955ff..d6b621fe3 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -10,7 +10,8 @@ 'id': '@', 'name': '@' }, - template: '
    • {{name}}
      • ', + transclude: true, + template: '
    • {{name}}
      • ', replace: true, link: function(scope) { scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) { diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index b5301a523..3bc3c751f 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -197,4 +197,15 @@ to add those CSS styles here. */ -webkit-border-radius: 6px 0 6px 6px; -moz-border-radius: 6px 0 6px 6px; border-radius: 6px 0 6px 6px; +} + +.rd-tab .remove { + cursor: pointer; + color: #A09797; + padding: 0 3px 1px 4px; +} +.rd-tab .remove:hover { + color: white; + background-color: #FF8080; + border-radius: 50%; } \ No newline at end of file diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 76dc905e6..ee00ffc12 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -54,8 +54,10 @@
        - - + + × + +
      From 1cd836ac8d038b0cbc316b6daacee03402881891 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 2 Feb 2014 18:20:18 +0200 Subject: [PATCH 141/540] Live visualization config POC (title only) --- rd_ui/app/scripts/directives.js | 33 ++++++- rd_ui/app/scripts/ng-highchart.js | 98 ++++++++++++--------- rd_ui/app/scripts/query_fiddle/renderers.js | 7 +- rd_ui/app/styles/redash.css | 1 + rd_ui/app/views/edit_visualization.html | 20 +---- rd_ui/app/views/queryfiddle.html | 8 +- 6 files changed, 97 insertions(+), 70 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index d6b621fe3..aae29c38b 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -61,6 +61,15 @@ return {type: type, name: Visualization.prototype.NAMES[type]}; }); + function getEditableOptions() { + return { + title: { + text: "New Chart" + } + }; + } + + if (!scope.vis) { // create new visualization // wait for query to load to populate with defaults @@ -72,15 +81,14 @@ 'type': scope.visTypes.CHART, 'name': q.name, 'description': q.description, - 'options': {} + 'options': getEditableOptions() }; } }, true); } scope.typeChanged = function() { - console.log('evme', 'typeChanged'); - scope.vis.options = {}; + scope.vis.options = getEditableOptions(); }; scope.submit = function() { @@ -324,6 +332,25 @@ }; }); + // http://stackoverflow.com/a/17904092/1559840 + directives.directive('jsonText', function() { + return { + restrict: 'A', + require: 'ngModel', + link: function(scope, element, attr, ngModel) { + function into(input) { + return JSON.parse(input); + } + function out(data) { + return JSON.stringify(data, undefined, 2); + } + ngModel.$parsers.push(into); + ngModel.$formatters.push(out); + + } + }; + }); + directives.directive('rdTimer', ['$timeout', function ($timeout) { return { restrict: 'E', diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index b4c445a29..ddd8b347d 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -22,60 +22,72 @@ angular.module('highchart', []) } }; - var deepCopy = true; - var newSettings = {}; - $.extend(deepCopy, newSettings, chartsDefaults, scope.options); + // Update when options change + scope.$watch('options', function(newOptions) { + initChart(newOptions); + }, true); + + //Update when charts data changes + scope.$watch(function () { + return (scope.series && scope.series.length) || 0; + }, function (length) { + if (!length || length == 0) { + scope.chart.showLoading(); + } else { + drawChart(); + }; + }, true); + + function initChart(options) { + if (scope.chart) { + scope.chart.destroy(); + } + + var deepCopy = true; + var newSettings = {}; + $.extend(deepCopy, newSettings, chartsDefaults, options); - // Making sure that the DOM is ready before creating the chart element, so it gets proper width. - $timeout(function(){ scope.chart = new Highcharts.Chart(newSettings); + drawChart(); + } - //Update when charts data changes - scope.$watch(function () { - return (scope.series && scope.series.length) || 0; - }, function (length) { - if (!length || length == 0) { - scope.chart.showLoading(); - } else { - while(scope.chart.series.length > 0) { - scope.chart.series[0].remove(true); - } + function drawChart() { + while(scope.chart.series.length > 0) { + scope.chart.series[0].remove(true); + } - if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { - scope.chart.xAxis[0].update({type: 'category'}); + if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { + scope.chart.xAxis[0].update({type: 'category'}); - // We need to make sure that for each category, each series has a value. - var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); + // We need to make sure that for each category, each series has a value. + var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); - _.each(scope.series, function(s) { - // TODO: move this logic to Query#getChartData - var yValues = _.groupBy(s.data, 'x'); + _.each(scope.series, function(s) { + // TODO: move this logic to Query#getChartData + var yValues = _.groupBy(s.data, 'x'); - var newData = _.sortBy(_.map(categories, function(category) { - return { - name: category, - y: yValues[category] && yValues[category][0].y - } - }), 'name'); + var newData = _.sortBy(_.map(categories, function(category) { + return { + name: category, + y: yValues[category] && yValues[category][0].y + } + }), 'name'); - s.data = newData; - }); - } else { - scope.chart.xAxis[0].update({type: 'datetime'}); - } + s.data = newData; + }); + } else { + scope.chart.xAxis[0].update({type: 'datetime'}); + } - scope.chart.counters.color = 0; + scope.chart.counters.color = 0; - _.each(scope.series, function(s) { - scope.chart.addSeries(s); - }) - - scope.chart.redraw(); - scope.chart.hideLoading(); - }; - }, true); - }); + _.each(scope.series, function(s) { + scope.chart.addSeries(s); + }) + scope.chart.redraw(); + scope.chart.hideLoading(); + } } }; diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 2fca58a1f..9b44ef1d3 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -90,8 +90,13 @@ renderers.directive('chartRenderer', function () { replace: false, controller: ['$scope', function ($scope) { $scope.chartSeries = []; - $scope.chartOptions = defaultChartOptions; + $scope.chartOptions = $.extend(true, {}, defaultChartOptions); + $scope.$watch('options', function(newOptions) { + if (newOptions) { + $.extend(true, $scope.chartOptions, newOptions); + } + }); $scope.$watch('queryResult && queryResult.getData()', function (data) { if (!data || $scope.queryResult.getData() == null) { $scope.chartSeries.splice(0, $scope.chartSeries.length); diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 3bc3c751f..f7d75aa9b 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -203,6 +203,7 @@ to add those CSS styles here. */ cursor: pointer; color: #A09797; padding: 0 3px 1px 4px; + font-size: 11px; } .rd-tab .remove:hover { color: white; diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html index 3c85498df..68ed4eded 100644 --- a/rd_ui/app/views/edit_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -17,25 +17,7 @@
      - -
      -

      - -

      -

      - -

      -
      - - -
      -

      - -

      -

      - -

      -
      +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index ee00ffc12..4a24f6d57 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -75,8 +75,8 @@
      - - + +

      @@ -89,8 +89,8 @@
      - - + +

      From 5ce3699a58382eb926e4b47a23f9ac2db1959c7e Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 3 Feb 2014 14:54:48 +0200 Subject: [PATCH 142/540] QueryFiddle: Live chart type editing --- rd_ui/app/scripts/directives.js | 46 ++-- rd_ui/app/scripts/ng-highchart.js | 246 +++++++++++++------- rd_ui/app/scripts/query_fiddle/renderers.js | 89 +------ rd_ui/app/scripts/services.js | 13 +- rd_ui/app/styles/redash.css | 4 + rd_ui/app/views/dashboard.html | 2 +- rd_ui/app/views/edit_visualization.html | 18 +- 7 files changed, 220 insertions(+), 198 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index aae29c38b..93a850afd 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -56,19 +56,15 @@ vis: '=?' }, link: function(scope, element, attrs) { - scope.visTypes = Visualization.prototype.TYPES; - scope.visOptions = _.map(Visualization.prototype.TYPES, function (type) { - return {type: type, name: Visualization.prototype.NAMES[type]}; - }); - - function getEditableOptions() { - return { - title: { - text: "New Chart" - } - }; - } - + scope.advancedMode = false; + scope.visTypes = { + 'Chart': Visualization.prototype.TYPES.CHART, + 'Cohort': Visualization.prototype.TYPES.COHORT + }; + scope.seriesTypes = { + 'Line': Visualization.prototype.SERIES_TYPES.LINE, + 'Bar': Visualization.prototype.SERIES_TYPES.BAR + }; if (!scope.vis) { // create new visualization @@ -78,17 +74,35 @@ unwatch(); scope.vis = { 'query_id': q.id, - 'type': scope.visTypes.CHART, + 'type': Visualization.prototype.TYPES.CHART, 'name': q.name, 'description': q.description, - 'options': getEditableOptions() + 'options': newOptions() }; } }, true); } + function newOptions(chartType) { + if (chartType === Visualization.prototype.TYPES.COHORT) { + // empty config at the moment + return {}; + } + + // Chart + return { + 'series': { + 'type': Visualization.prototype.SERIES_TYPES.LINE + } + }; + } + + scope.toggleAdvancedMode = function() { + scope.advancedMode = !scope.advancedMode; + }; + scope.typeChanged = function() { - scope.vis.options = getEditableOptions(); + scope.vis.options = newOptions(); }; scope.submit = function() { diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index ddd8b347d..701a23137 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -1,95 +1,181 @@ -'use strict'; +(function(){ + 'use strict'; -angular.module('highchart', []) - .directive('chart', ['$timeout', function ($timeout) { - return { - restrict: 'E', - template: '
      ', - scope: { - options: "=options", - series: "=series" - }, - transclude: true, - replace: true, + var defaultOptions = { + title: { + "text": null + }, + tooltip: { + valueDecimals: 2, + formatter: function () { + if (moment.isMoment(this.x)) { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; - link: function (scope, element, attrs) { - var chartsDefaults = { - chart: { - renderTo: element[0], - type: attrs.type || null, - height: attrs.height || null, - width: attrs.width || null - } - }; + $.each(this.points, function (i, point) { + s += '
      ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); - // Update when options change - scope.$watch('options', function(newOptions) { - initChart(newOptions); - }, true); - - //Update when charts data changes - scope.$watch(function () { - return (scope.series && scope.series.length) || 0; - }, function (length) { - if (!length || length == 0) { - scope.chart.showLoading(); - } else { - drawChart(); - }; - }, true); - - function initChart(options) { - if (scope.chart) { - scope.chart.destroy(); - } - - var deepCopy = true; - var newSettings = {}; - $.extend(deepCopy, newSettings, chartsDefaults, options); - - scope.chart = new Highcharts.Chart(newSettings); - drawChart(); + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; + } + }); + } else { + var s = "" + this.points[0].key + ""; + $.each(this.points, function (i, point) { + s+= '
      ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + }); } - function drawChart() { - while(scope.chart.series.length > 0) { - scope.chart.series[0].remove(true); + return s; + }, + shared: true + }, + xAxis: { + type: 'datetime' + }, + yAxis: { + title: { + text: null + } + }, + exporting: { + chartOptions: { + title: { + text: '' + } + }, + buttons: { + contextButton: { + menuItems: [ + { + text: 'Toggle % Stacking', + onclick: function () { + var newStacking = "normal"; + if (this.series[0].options.stacking == "normal") { + newStacking = "percent"; + } + + _.each(this.series, function (series) { + series.update({stacking: newStacking}, true); + }); + } + } + ] + } + } + }, + credits: { + enabled: false + }, + plotOptions: { + "column": { + "stacking": "normal", + "pointPadding": 0, + "borderWidth": 1, + "groupPadding": 0, + "shadow": false + } + }, + series: [] + }; + + angular.module('highchart', []) + .directive('chart', ['$timeout', function ($timeout) { + return { + restrict: 'E', + template: '
      ', + scope: { + options: "=options", + series: "=series" + }, + transclude: true, + replace: true, + + link: function (scope, element, attrs) { + var chartsDefaults = { + chart: { + renderTo: element[0], + type: attrs.type || null, + height: attrs.height || null, + width: attrs.width || null + } + }; + + var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults); + + // Update when options change + scope.$watch('options', function(newOptions) { + initChart(newOptions); + }, true); + + //Update when charts data changes + scope.$watch(function () { + return (scope.series && scope.series.length) || 0; + }, function (length) { + if (!length || length == 0) { + scope.chart.showLoading(); + } else { + drawChart(); + }; + }, true); + + function initChart(options) { + if (scope.chart) { + scope.chart.destroy(); + } + + $.extend(true, chartOptions, options); + + scope.chart = new Highcharts.Chart(chartOptions); + drawChart(); } - if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { - scope.chart.xAxis[0].update({type: 'category'}); + function drawChart() { + while(scope.chart.series.length > 0) { + scope.chart.series[0].remove(true); + } - // We need to make sure that for each category, each series has a value. - var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); + // todo series.type + + if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { + scope.chart.xAxis[0].update({type: 'category'}); + + // We need to make sure that for each category, each series has a value. + var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); + + _.each(scope.series, function(s) { + // TODO: move this logic to Query#getChartData + var yValues = _.groupBy(s.data, 'x'); + + var newData = _.sortBy(_.map(categories, function(category) { + return { + name: category, + y: yValues[category] && yValues[category][0].y + } + }), 'name'); + + s.data = newData; + }); + } else { + scope.chart.xAxis[0].update({type: 'datetime'}); + } + + scope.chart.counters.color = 0; _.each(scope.series, function(s) { - // TODO: move this logic to Query#getChartData - var yValues = _.groupBy(s.data, 'x'); + // here we override the series with the visualization config + var _s = $.extend(true, {}, s, chartOptions['series']); + scope.chart.addSeries(_s); + }) - var newData = _.sortBy(_.map(categories, function(category) { - return { - name: category, - y: yValues[category] && yValues[category][0].y - } - }), 'name'); - - s.data = newData; - }); - } else { - scope.chart.xAxis[0].update({type: 'datetime'}); + scope.chart.redraw(); + scope.chart.hideLoading(); } - scope.chart.counters.color = 0; - - _.each(scope.series, function(s) { - scope.chart.addSeries(s); - }) - - scope.chart.redraw(); - scope.chart.hideLoading(); } + }; - } - }; - - }]); \ No newline at end of file + }]); +})(); \ No newline at end of file diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 9b44ef1d3..a654f5876 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -1,83 +1,4 @@ var renderers = angular.module('redash.renderers', []); -var defaultChartOptions = { - "title": { - "text": null - }, - "tooltip": { - valueDecimals: 2, - formatter: function () { - if (moment.isMoment(this.x)) { - var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', - pointsCount = this.points.length; - - $.each(this.points, function (i, point) { - s += '
      ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - - if (pointsCount > 1 && point.percentage) { - s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; - } - }); - } else { - var s = "" + this.points[0].key + ""; - $.each(this.points, function (i, point) { - s+= '
      ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - }); - } - - return s; - }, - shared: true - }, - xAxis: { - type: 'datetime' - }, - yAxis: { - title: { - text: null - } - }, - exporting: { - chartOptions: { - title: { - text: this.description - } - }, - buttons: { - contextButton: { - menuItems: [ - { - text: 'Toggle % Stacking', - onclick: function () { - var newStacking = "normal"; - if (this.series[0].options.stacking == "normal") { - newStacking = "percent"; - } - - _.each(this.series, function (series) { - series.update({stacking: newStacking}, true); - }); - } - } - ] - } - } - }, - credits: { - enabled: false - }, - plotOptions: { - "column": { - "stacking": "normal", - "pointPadding": 0, - "borderWidth": 1, - "groupPadding": 0, - "shadow": false - } - }, - "series": [] -}; renderers.directive('chartRenderer', function () { return { @@ -90,11 +11,11 @@ renderers.directive('chartRenderer', function () { replace: false, controller: ['$scope', function ($scope) { $scope.chartSeries = []; - $scope.chartOptions = $.extend(true, {}, defaultChartOptions); + $scope.chartOptions = {}; - $scope.$watch('options', function(newOptions) { - if (newOptions) { - $.extend(true, $scope.chartOptions, newOptions); + $scope.$watch('options', function(chartOptions) { + if (chartOptions) { + $scope.chartOptions = chartOptions; } }); $scope.$watch('queryResult && queryResult.getData()', function (data) { @@ -104,7 +25,7 @@ renderers.directive('chartRenderer', function () { $scope.chartSeries.splice(0, $scope.chartSeries.length); _.each($scope.queryResult.getChartData(), function (s) { - $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}, $scope.options)); + $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'})); }); } }); diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 919804208..0fc4ac7aa 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -290,19 +290,16 @@ }; var Visualization = function($resource) { - var CHART = 'CHART'; - var COHORT = 'COHORT'; - var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); Visualization.prototype = { TYPES: { - 'CHART': CHART, - 'COHORT': COHORT + 'CHART': 'CHART', + 'COHORT': 'COHORT' }, - NAMES: { - 'CHART': 'Chart', - 'COHORT': 'Cohort' + SERIES_TYPES: { + 'LINE': 'line', + 'BAR': 'bar' } }; diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index f7d75aa9b..2da630180 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -2,6 +2,10 @@ body { padding-top: 70px; } +a.link { + cursor: pointer; +} + a.page-title { overflow: hidden; text-overflow: ellipsis; diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 1fa941d30..c148db147 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -30,7 +30,7 @@
      - +
      diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html index 68ed4eded..a70be6775 100644 --- a/rd_ui/app/views/edit_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -10,22 +10,22 @@
      - - + + +
      + +
      + +
      - - - - + Advanced Mode +
      -
      - {{vis}} -
      From a45ba0bf309d66946d7f93ae9101887d59f30f48 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 3 Feb 2014 16:12:29 +0200 Subject: [PATCH 143/540] Dashboard visualizations --- rd_service/data/models.py | 2 +- rd_ui/app/scripts/controllers.js | 4 +- rd_ui/app/scripts/directives.js | 26 ++++++------ rd_ui/app/scripts/query_fiddle/renderers.js | 16 +++++++ rd_ui/app/scripts/services.js | 6 --- rd_ui/app/views/dashboard.html | 7 +-- rd_ui/app/views/edit_visualization.html | 2 +- rd_ui/app/views/new_widget_form.html | 47 +++++---------------- rd_ui/app/views/queryfiddle.html | 10 ++--- 9 files changed, 50 insertions(+), 70 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index 7ca0abfb1..882144448 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -201,7 +201,7 @@ class Widget(models.Model): 'type': self.type, 'width': self.width, 'options': json.loads(self.options), - 'visualization_id': self.visualization_id, + 'visualization': self.visualization.to_dict(), 'dashboard_id': self.dashboard_id } diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index f7c7ff979..9ec09e6e7 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -7,7 +7,7 @@ var WidgetCtrl = function ($scope, $http, $location, Query) { $scope.deleteWidget = function() { - if (!confirm('Are you sure you want to remove "' + $scope.widget.query.name + '" from the dashboard?')) { + if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) { return; } @@ -24,7 +24,7 @@ $location.path('/queries/' + query.id); } - $scope.query = new Query($scope.widget.query); + $scope.query = new Query($scope.widget.visualization.query); $scope.queryResult = $scope.query.getQueryResult(); $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString(); diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 93a850afd..d55913e14 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -70,7 +70,7 @@ // create new visualization // wait for query to load to populate with defaults var unwatch = scope.$watch('query', function(q) { - if (q.id) { + if (q && q.id) { unwatch(); scope.vis = { 'query_id': q.id, @@ -168,7 +168,7 @@ row: rowIndex+1, ySize: 1, xSize: widget.width, - name: widget.query.name + name: widget.visualization.name }); }); }); @@ -230,14 +230,14 @@ templateUrl: '/views/new_widget_form.html', replace: true, link: function($scope, element, attrs) { - $scope.visReady = false; - $scope.currentView = 'existing'; $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}]; var reset = function() { $scope.saveInProgress = false; $scope.widgetSize = 1; $scope.queryId = null; + $scope.selectedVis = null; + } reset(); @@ -247,18 +247,18 @@ }; $scope.loadVisualizations = function() { + if (!$scope.queryId) { + return; + } + Query.get({ id: $scope.queryId }, function(query) { if (query) { - // TODO - $scope.visReady = true; - var visualizations = query.getVisualizations(); - $scope.existing = visualizations = [ - {'name': 'vis1', 'type': 'Cohort'}, - {'name': 'vis2', 'type': 'Pivot Table'} - ]; - $scope.currentView = (visualizations.length) ? 'existing' : 'addNew'; + $scope.query = query; + if(query.visualizations.length) { + $scope.selectedVis = query.visualizations[0]; + } } }); }; @@ -267,7 +267,7 @@ $scope.saveInProgress = true; var widget = { - 'query_id': $scope.queryId, + 'visualization_id': $scope.selectedVis.id, 'dashboard_id': $scope.dashboard.id, 'options': {}, 'width': $scope.widgetSize diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index a654f5876..9ee3d0502 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -1,5 +1,21 @@ var renderers = angular.module('redash.renderers', []); +renderers.directive('visualizationRenderer', function() { + return { + restrict: 'E', + scope: { + visualization: '=', + queryResult: '=' + }, + template: '
      ' + + '' + + '' + + '' + + '
      ', + replace: false + } +}); + renderers.directive('chartRenderer', function () { return { restrict: 'E', diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 0fc4ac7aa..5356e4595 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -280,12 +280,6 @@ return [this.name, this.description, this.query].join('!#'); }; - Query.prototype.getVisualizations = function() { - // TODO - // not implemented - return []; - }; - return Query; }; diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index c148db147..c17800441 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -29,11 +29,8 @@
      -
      - - - -
      + +
      - Advanced Mode + Advanced
      diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 50a52e297..88ce57afb 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -9,50 +9,25 @@

      - +
      + Load +

      -
      +
      -
      - -
      -
      - - Existing - -
      -
      -
      -
        -
      • - {{item.name}} {{item.type}} -
        • -
      -
      -
      -
      + +
      - -
      -
      - - Add New - -
      -
      -
      - -
      -
      -
      -
      +
      + + Add New +
      + +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 4a24f6d57..fd6915f87 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -74,9 +74,8 @@
      -
      - - +
      +

      @@ -88,9 +87,8 @@
      -
      - - +
      +

      From 7b78bfe191c7716cd7793548cad9c49ce3b05cb6 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 3 Feb 2014 16:35:16 +0200 Subject: [PATCH 144/540] add Visualization and SERIES types --- rd_ui/app/scripts/directives.js | 6 ++++-- rd_ui/app/scripts/services.js | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index d55913e14..c01a6923e 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -59,11 +59,13 @@ scope.advancedMode = false; scope.visTypes = { 'Chart': Visualization.prototype.TYPES.CHART, - 'Cohort': Visualization.prototype.TYPES.COHORT + 'Cohort': Visualization.prototype.TYPES.COHORT, + 'Table': Visualization.prototype.TYPES.GRID }; scope.seriesTypes = { 'Line': Visualization.prototype.SERIES_TYPES.LINE, - 'Bar': Visualization.prototype.SERIES_TYPES.BAR + 'Bar': Visualization.prototype.SERIES_TYPES.BAR, + 'Area': Visualization.prototype.SERIES_TYPES.AREA }; if (!scope.vis) { diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 5356e4595..115b096f8 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -289,11 +289,13 @@ Visualization.prototype = { TYPES: { 'CHART': 'CHART', - 'COHORT': 'COHORT' + 'COHORT': 'COHORT', + 'GRID': 'GRID' }, SERIES_TYPES: { 'LINE': 'line', - 'BAR': 'bar' + 'BAR': 'bar', + 'AREA': 'area' } }; From bb096be00c05362a0c6ce3e3f50838329e23325c Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 4 Feb 2014 15:16:07 +0200 Subject: [PATCH 145/540] Visualization.name length to 255 (should match Query.name length) --- rd_service/data/models.py | 2 +- rd_service/data/tables.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index 882144448..c771a7de8 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -157,7 +157,7 @@ class Visualization(models.Model): id = models.AutoField(primary_key=True) type = models.CharField(max_length=100) query = models.ForeignKey(Query, related_name='visualizations') - name = models.CharField(max_length=100) + name = models.CharField(max_length=255) description = models.CharField(max_length=4096) options = models.TextField() diff --git a/rd_service/data/tables.sql b/rd_service/data/tables.sql index 4e1f24ded..24cac5c34 100644 --- a/rd_service/data/tables.sql +++ b/rd_service/data/tables.sql @@ -34,7 +34,7 @@ CREATE TABLE "visualizations" ( "id" serial NOT NULL PRIMARY KEY, "type" varchar(100) NOT NULL, "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, - "name" varchar(100) NOT NULL, + "name" varchar(255) NOT NULL, "description" varchar(4096), "options" text NOT NULL ) From cd838e5a7e50aea25a60a0d5c7dca95c38963067 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 4 Feb 2014 16:11:48 +0200 Subject: [PATCH 146/540] migrating Widgets to Visualizations --- rd_service/data/models.py | 1 + rd_service/migrate.py | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 rd_service/migrate.py diff --git a/rd_service/data/models.py b/rd_service/data/models.py index c771a7de8..b63da9fa5 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -186,6 +186,7 @@ class Visualization(models.Model): class Widget(models.Model): id = models.AutoField(primary_key=True) type = models.CharField(max_length=100) + query = models.ForeignKey(Query, related_name='widgets') visualization = models.ForeignKey(Visualization, related_name='widgets') width = models.IntegerField() options = models.TextField() diff --git a/rd_service/migrate.py b/rd_service/migrate.py new file mode 100644 index 000000000..9444e156e --- /dev/null +++ b/rd_service/migrate.py @@ -0,0 +1,50 @@ +import json +import settings +from data.models import * + +# first run: + +# CREATE TABLE "visualizations" ( +# "id" serial NOT NULL PRIMARY KEY, +# "type" varchar(100) NOT NULL, +# "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, +# "name" varchar(255) NOT NULL, +# "description" varchar(4096), +# "options" text NOT NULL +# ) +# ; + +# ALTER TABLE widgets ADD COLUMN "visualization_id" integer REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED; + +if __name__ == '__main__': + print 'migrating Widgets -> Visualizations ...' + + for query in Query.objects.filter(name__icontains="cohort"): + vis = Visualization(query=query, name=query.name, + description=query.description, + type="COHORT", options="{}") + vis.save() + + + for widget in Widget.objects.all(): + print 'processing widget %d' % widget.id + query = widget.query + vis_type = widget.type.upper() + + vis = query.visualizations.filter(type=vis_type) + if vis: + print 'found' + widget.visualization = vis[0] + widget.save() + + else: + options = json.loads(widget.options) + vis_options = {"series": options} if options else {} + vis_options = json.dumps(vis_options) + + vis = Visualization(query=query, name=query.name, + description=query.description, + type=vis_type, options=vis_options) + vis.save() + widget.visualization = vis + widget.save() \ No newline at end of file From 5b532d03a089400bdf59d7f23deb3ca211fe9e8a Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 4 Feb 2014 16:56:35 +0200 Subject: [PATCH 147/540] version bump 0.2 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ddd53e6d0..996de5fa2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ NAME=redash -VERSION=0.1 +VERSION=0.2 FULL_VERSION=$(VERSION).$(CIRCLE_BUILD_NUM) FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz From 567a732e1ead57b371bcb4f983b9fcd086f59a54 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Feb 2014 16:58:36 +0200 Subject: [PATCH 148/540] Readme update (added reference to mailing list & IRC channel) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 921d062ef..64e277d31 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,11 @@ You can try out the demo instance: http://rd-demo.herokuapp.com/ (login with any Due to Heroku dev plan limits, it has a small database of flights (see schema [here](http://rd-demo.herokuapp.com/dashboard/schema)). Also due to another Heroku limitation, it is running with the regular user, hence you can DELETE or INSERT data/tables. Please be nice and don't do this. +## Getting help + +* [Google Group (mailing list)](https://groups.google.com/forum/#!forum/redash-users): -- the best place to get updates about new releases or ask general questions. +* #redash IRC channel on [Freenode](http://www.freenode.net/). + ## Technology * [AngularJS](http://angularjs.org/) From 97ca722a11621b3b999ec7c084037037ad6445a9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Feb 2014 17:02:24 +0200 Subject: [PATCH 149/540] Small fix to README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64e277d31..ee5b67cf8 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Due to Heroku dev plan limits, it has a small database of flights (see schema [h ## Getting help -* [Google Group (mailing list)](https://groups.google.com/forum/#!forum/redash-users): -- the best place to get updates about new releases or ask general questions. +* [Google Group (mailing list)](https://groups.google.com/forum/#!forum/redash-users): the best place to get updates about new releases or ask general questions. * #redash IRC channel on [Freenode](http://www.freenode.net/). ## Technology From 3360cd934b971e34f4e892520a25f025ff59f371 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 09:39:00 +0200 Subject: [PATCH 150/540] Remove backward compatability workaround (fixes #75) --- rd_ui/app/scripts/query_fiddle/renderers.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 2fca58a1f..ff3a071a5 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -142,13 +142,7 @@ renderers.directive('gridRenderer', function () { var gridData = _.map($scope.queryResult.getData(), function (row) { var newRow = {}; _.each(row, function (val, key) { - // TODO: hack to detect date fields, needed only for backward compatability - if (val > 1000 * 1000 * 1000 * 100) { - newRow[$scope.queryResult.getColumnCleanName(key)] = moment(val); - } else { - newRow[$scope.queryResult.getColumnCleanName(key)] = val; - } - + newRow[$scope.queryResult.getColumnCleanName(key)] = val; }) return newRow; }); @@ -264,4 +258,4 @@ renderers.directive('cohortRenderer', function() { }); } } -}) \ No newline at end of file +}) From 212ade2da7970183fcf3551aa25ed9a9fd7ed942 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 09:47:51 +0200 Subject: [PATCH 151/540] Add query_id back to widgets in tables.sql until we remove it from Model --- rd_service/data/tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rd_service/data/tables.sql b/rd_service/data/tables.sql index 24cac5c34..5fb00a68d 100644 --- a/rd_service/data/tables.sql +++ b/rd_service/data/tables.sql @@ -44,6 +44,7 @@ CREATE TABLE "widgets" ( "type" varchar(100) NOT NULL, "width" integer NOT NULL, "options" text NOT NULL, + "query_id" integer, "visualization_id" integer NOT NULL REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED, "dashboard_id" integer NOT NULL REFERENCES "dashboards" ("id") DEFERRABLE INITIALLY DEFERRED ) @@ -52,4 +53,4 @@ CREATE INDEX "queries_latest_query_data_id" ON "queries" ("latest_query_data_id" CREATE INDEX "widgets_query_id" ON "widgets" ("query_id"); CREATE INDEX "widgets_dashboard_id" ON "widgets" ("dashboard_id"); -COMMIT; \ No newline at end of file +COMMIT; From a9e135c94f6c8692dc9c8a83589ef791691d3ba1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 09:48:11 +0200 Subject: [PATCH 152/540] Exclude venv dir from the release package --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 996de5fa2..dc8ed84e6 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ deps: cd rd_ui && grunt build pack: - tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * + tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: python bin/upload_version.py $(FULL_VERSION) $(FILENAME) From b51ef059f5c0706e2a4bf80b6df15f34e85351cf Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 5 Feb 2014 17:54:08 +0200 Subject: [PATCH 153/540] fix migration to set 'bars' as default --- rd_service/migrate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rd_service/migrate.py b/rd_service/migrate.py index 9444e156e..caeff82f1 100644 --- a/rd_service/migrate.py +++ b/rd_service/migrate.py @@ -19,6 +19,8 @@ from data.models import * if __name__ == '__main__': print 'migrating Widgets -> Visualizations ...' + default_options = {"series": {"type": "bar"}} + for query in Query.objects.filter(name__icontains="cohort"): vis = Visualization(query=query, name=query.name, description=query.description, @@ -39,7 +41,7 @@ if __name__ == '__main__': else: options = json.loads(widget.options) - vis_options = {"series": options} if options else {} + vis_options = {"series": options} if options else default_options vis_options = json.dumps(vis_options) vis = Visualization(query=query, name=query.name, From 9aaf17d478b5830cf25dd89a7d56ecba938611c8 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 6 Feb 2014 16:35:29 +0200 Subject: [PATCH 154/540] Fixes #80: * Create default 'Table' visualization for all queries * remove 'Table' type when creating new visualization * Set type as the default visualization name (instead of the query name) * Remove description field and advanced mode * Remove section for adding new visualization in new widget dialog --- rd_service/data/models.py | 6 ++++ rd_service/migrate.py | 31 +++++++++++++++------ rd_service/server.py | 1 + rd_ui/app/scripts/directives.js | 17 ++++++----- rd_ui/app/scripts/query_fiddle/renderers.js | 1 - rd_ui/app/scripts/services.js | 2 +- rd_ui/app/views/edit_visualization.html | 12 +------- rd_ui/app/views/new_widget_form.html | 7 ----- rd_ui/app/views/queryfiddle.html | 5 ++-- 9 files changed, 45 insertions(+), 37 deletions(-) diff --git a/rd_service/data/models.py b/rd_service/data/models.py index b63da9fa5..c56ef3eda 100644 --- a/rd_service/data/models.py +++ b/rd_service/data/models.py @@ -51,6 +51,12 @@ class Query(models.Model): app_label = 'redash' db_table = 'queries' + def create_default_visualizations(self): + table_visualization = Visualization(query=self, name="Table", + description=self.description, + type="TABLE", options="{}") + table_visualization.save() + def to_dict(self, with_result=True, with_stats=False, with_visualizations=False): d = { diff --git a/rd_service/migrate.py b/rd_service/migrate.py index caeff82f1..2930b7b4a 100644 --- a/rd_service/migrate.py +++ b/rd_service/migrate.py @@ -17,36 +17,51 @@ from data.models import * # ALTER TABLE widgets ADD COLUMN "visualization_id" integer REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED; if __name__ == '__main__': - print 'migrating Widgets -> Visualizations ...' - default_options = {"series": {"type": "bar"}} + # create 'table' visualization for all queries + print 'creating TABLE visualizations ...' + for query in Query.objects.all(): + vis = Visualization(query=query, name="Table", + description=query.description, + type="TABLE", options="{}") + vis.save() + + + # create 'cohort' visualization for all queries named with 'cohort' + print 'creating COHORT visualizations ...' for query in Query.objects.filter(name__icontains="cohort"): - vis = Visualization(query=query, name=query.name, + vis = Visualization(query=query, name="Cohort", description=query.description, type="COHORT", options="{}") vis.save() + # create visualization for every widget (unless it already exists) + print 'migrating Widgets -> Visualizations ...' for widget in Widget.objects.all(): - print 'processing widget %d' % widget.id + print 'processing widget %d:' % widget.id, query = widget.query vis_type = widget.type.upper() vis = query.visualizations.filter(type=vis_type) if vis: - print 'found' + print 'visualization exists' widget.visualization = vis[0] widget.save() else: + vis_name = widget.type.title() + options = json.loads(widget.options) vis_options = {"series": options} if options else default_options vis_options = json.dumps(vis_options) - vis = Visualization(query=query, name=query.name, - description=query.description, - type=vis_type, options=vis_options) + vis = Visualization(query=query, name=vis_name, + description=query.description, + type=vis_type, options=vis_options) + + print 'created visualization %s' % vis_type vis.save() widget.visualization = vis widget.save() \ No newline at end of file diff --git a/rd_service/server.py b/rd_service/server.py index 5b09079eb..edd234cee 100644 --- a/rd_service/server.py +++ b/rd_service/server.py @@ -215,6 +215,7 @@ class QueriesHandler(BaseAuthenticatedHandler): query_def['user'] = self.current_user query = data.models.Query(**query_def) query.save() + query.create_default_visualizations() self.write_json(query.to_dict(with_result=False)) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index c01a6923e..beb9e675d 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -59,8 +59,7 @@ scope.advancedMode = false; scope.visTypes = { 'Chart': Visualization.prototype.TYPES.CHART, - 'Cohort': Visualization.prototype.TYPES.COHORT, - 'Table': Visualization.prototype.TYPES.GRID + 'Cohort': Visualization.prototype.TYPES.COHORT }; scope.seriesTypes = { 'Line': Visualization.prototype.SERIES_TYPES.LINE, @@ -77,7 +76,7 @@ scope.vis = { 'query_id': q.id, 'type': Visualization.prototype.TYPES.CHART, - 'name': q.name, + 'name': '', 'description': q.description, 'options': newOptions() }; @@ -99,6 +98,14 @@ }; } + scope.$watch('vis.type', function(type) { + // if not edited by user, set name to match type + if (type && scope.vis && !scope.visForm.name.$dirty) { + // poor man's titlecase + scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase(); + } + }); + scope.toggleAdvancedMode = function() { scope.advancedMode = !scope.advancedMode; }; @@ -244,10 +251,6 @@ reset(); - $scope.toggleView = function(viewName) { - $scope.currentView = ($scope.currentView == viewName) ? '' : viewName; - }; - $scope.loadVisualizations = function() { if (!$scope.queryId) { return; diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 9471e0283..786442cb6 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -9,7 +9,6 @@ renderers.directive('visualizationRenderer', function() { }, template: '
      ' + '' + - '' + '' + '
      ', replace: false diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 115b096f8..092412b7b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -290,7 +290,7 @@ TYPES: { 'CHART': 'CHART', 'COHORT': 'COHORT', - 'GRID': 'GRID' + 'TABLE': 'TABLE' }, SERIES_TYPES: { 'LINE': 'line', diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html index d7c8b8403..bcba4a02e 100644 --- a/rd_ui/app/views/edit_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -1,12 +1,7 @@
      - -
      - -
      - - +
      @@ -19,11 +14,6 @@
      -
      - Advanced - -
      -
      diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 88ce57afb..41a1e0299 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -23,13 +23,6 @@
      -
      - + Add New -
      - -
      -
      -
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index fd6915f87..8b4e60d6b 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -54,7 +54,8 @@
        - + + × @@ -83,7 +84,7 @@
        -

        +

        From 6386f0f9aa1e2f9f930a274d021a213a54b8ed39 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 6 Feb 2014 16:40:58 +0200 Subject: [PATCH 155/540] fix issue where start_workers failed when settings.CONNECTION_ADAPTER does not exist --- rd_service/data/manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rd_service/data/manager.py b/rd_service/data/manager.py index d4df6d137..52d2dd54b 100644 --- a/rd_service/data/manager.py +++ b/rd_service/data/manager.py @@ -153,12 +153,12 @@ class Manager(object): def start_workers(self, workers_count, connection_string): if self.workers: return self.workers - - if settings.CONNECTION_ADAPTER == "mysql": - import query_runner_mysql + + if getattr(settings, 'CONNECTION_ADAPTER', None) == "mysql": + import query_runner_mysql runner = query_runner_mysql.mysql(connection_string) else: - import query_runner + import query_runner runner = query_runner.redshift(connection_string) redis_connection_params = self.redis_connection.connection_pool.connection_kwargs From d84d047470f30ff0ddf8fd4cdc99857ed34b5abe Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:04:31 +0200 Subject: [PATCH 156/540] Rename rd_service to redash. --- .gitignore | 4 ++-- {rd_service => redash}/__init__.py | 0 {rd_service => redash}/cli.py | 0 {rd_service => redash}/data/__init__.py | 0 {rd_service => redash}/data/manager.py | 0 {rd_service => redash}/data/models.py | 0 {rd_service => redash}/data/query_runner.py | 0 {rd_service => redash}/data/tables.sql | 0 {rd_service => redash}/data/utils.py | 0 {rd_service => redash}/data/worker.py | 0 {rd_service => redash}/requirements.txt | 0 {rd_service => redash}/server.py | 0 {rd_service => redash}/settings_example.py | 0 13 files changed, 2 insertions(+), 2 deletions(-) rename {rd_service => redash}/__init__.py (100%) rename {rd_service => redash}/cli.py (100%) rename {rd_service => redash}/data/__init__.py (100%) rename {rd_service => redash}/data/manager.py (100%) rename {rd_service => redash}/data/models.py (100%) rename {rd_service => redash}/data/query_runner.py (100%) rename {rd_service => redash}/data/tables.sql (100%) rename {rd_service => redash}/data/utils.py (100%) rename {rd_service => redash}/data/worker.py (100%) rename {rd_service => redash}/requirements.txt (100%) rename {rd_service => redash}/server.py (100%) rename {rd_service => redash}/settings_example.py (100%) diff --git a/.gitignore b/.gitignore index 059ee4bdd..32e64b015 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ .idea *.pyc -rd_service/settings.py +redash/settings.py rd_ui/dist .DS_Store # Vagrant related .vagrant Berksfile.lock -rd_service/dump.rdb +redash/dump.rdb diff --git a/rd_service/__init__.py b/redash/__init__.py similarity index 100% rename from rd_service/__init__.py rename to redash/__init__.py diff --git a/rd_service/cli.py b/redash/cli.py similarity index 100% rename from rd_service/cli.py rename to redash/cli.py diff --git a/rd_service/data/__init__.py b/redash/data/__init__.py similarity index 100% rename from rd_service/data/__init__.py rename to redash/data/__init__.py diff --git a/rd_service/data/manager.py b/redash/data/manager.py similarity index 100% rename from rd_service/data/manager.py rename to redash/data/manager.py diff --git a/rd_service/data/models.py b/redash/data/models.py similarity index 100% rename from rd_service/data/models.py rename to redash/data/models.py diff --git a/rd_service/data/query_runner.py b/redash/data/query_runner.py similarity index 100% rename from rd_service/data/query_runner.py rename to redash/data/query_runner.py diff --git a/rd_service/data/tables.sql b/redash/data/tables.sql similarity index 100% rename from rd_service/data/tables.sql rename to redash/data/tables.sql diff --git a/rd_service/data/utils.py b/redash/data/utils.py similarity index 100% rename from rd_service/data/utils.py rename to redash/data/utils.py diff --git a/rd_service/data/worker.py b/redash/data/worker.py similarity index 100% rename from rd_service/data/worker.py rename to redash/data/worker.py diff --git a/rd_service/requirements.txt b/redash/requirements.txt similarity index 100% rename from rd_service/requirements.txt rename to redash/requirements.txt diff --git a/rd_service/server.py b/redash/server.py similarity index 100% rename from rd_service/server.py rename to redash/server.py diff --git a/rd_service/settings_example.py b/redash/settings_example.py similarity index 100% rename from rd_service/settings_example.py rename to redash/settings_example.py From b31c5be70e43fc6809ed19d5a76928aeffebdbc8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:05:54 +0200 Subject: [PATCH 157/540] Replace Tornado with Flask --- rd_ui/app/index.html | 12 +- redash/api.py | 342 ++++++++++++++++++++++++++++++++++++++ redash/server.py | 382 ------------------------------------------- 3 files changed, 349 insertions(+), 387 deletions(-) create mode 100644 redash/api.py delete mode 100644 redash/server.py diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index d4877bbbb..a3964f749 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -31,6 +31,7 @@ re:dash
        + {% raw %}

          - {{!currentUser.name}} + {{currentUser.name}}

        + {% endraw %}
        @@ -119,13 +121,13 @@ diff --git a/redash/api.py b/redash/api.py new file mode 100644 index 000000000..3ccde869e --- /dev/null +++ b/redash/api.py @@ -0,0 +1,342 @@ +""" +Flask-restful based API implementation for re:dash. + +Also at the moment the Flask server is used to serve the static assets (and the Angular.js app), +but this is only due to configuration issues and temporary. + +Usage: + python api.py [--port=8888] [--debug] [--static=..] + + port - port to listen to + debug - enable debug mode (extensive logging, restart on code change) + static - static assets path + +If static option isn't specified it will be taken from settings.py. +""" +import csv +import hashlib +import json +import numbers +import urlparse +import cStringIO +import datetime +import dateutil + +from flask import Flask, g, render_template, send_from_directory, make_response, request, jsonify +from flask_googleauth import GoogleFederated +from flask.ext.restful import Api, Resource, abort + +import redis +import sqlparse +import settings +from data import utils +import data + +app = Flask(__name__, + template_folder=settings.STATIC_ASSETS_PATH, + static_folder=settings.STATIC_ASSETS_PATH, + static_path='/static') + +api = Api(app) + +@api.representation('application/json') +def json_representation(data, code, headers=None): + resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code) + resp.headers.extend(headers or {}) + return resp + + +# TODO: move this out +url = urlparse.urlparse(settings.REDIS_URL) +redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password) +data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) + +auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) +from werkzeug.contrib.fixers import ProxyFix +app.wsgi_app = ProxyFix(app.wsgi_app) +app.secret_key = settings.COOKIE_SECRET +auth.force_auth_on_every_request = True + + +@app.route('/ping', methods=['GET']) +def ping(): + return 'PONG.' + + +@app.route('/admin/') +@app.route('/dashboard/') +@app.route('/queries/') +@app.route('/') +@auth.required +def index(anything=None): + email_md5 = hashlib.md5(g.user['email'].lower()).hexdigest() + gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5 + + user = { + 'gravatar_url': gravatar_url, + 'is_admin': g.user['email'] in settings.ADMINS, + 'name': g.user['email'] + } + + return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) + + +@app.route('/status.json') +@auth.required +def status_api(): + status = {} + info = redis_connection.info() + status['redis_used_memory'] = info['used_memory_human'] + + status['queries_count'] = data.models.Query.objects.count() + status['query_results_count'] = data.models.QueryResult.objects.count() + status['dashboards_count'] = data.models.Dashboard.objects.count() + status['widgets_count'] = data.models.Widget.objects.count() + + status['workers'] = [redis_connection.hgetall(w) + for w in redis_connection.smembers('workers')] + + manager_status = redis_connection.hgetall('manager:status') + status['manager'] = manager_status + status['manager']['queue_size'] = redis_connection.zcard('jobs') + + return jsonify(status) + + +@app.route('/api/queries/format', methods=['POST']) +@auth.required +def format_sql_query(self): + arguments = json.loads(self.request.body) + query = arguments.get("query", "") + + return sqlparse.format(query, reindent=True, keyword_case='upper') + + +class BaseResource(Resource): + decorators = [auth.required] + + @property + def current_user(self): + return g.user['email'] + + +class DashboardListAPI(BaseResource): + def get(self): + dashboards = [d.to_dict() for d in + data.models.Dashboard.objects.filter(is_archived=False)] + + return dashboards + + def post(self): + dashboard_properties = json.loads(self.request.body) + dashboard = data.models.Dashboard(name=dashboard_properties['name'], + user=self.current_user, + layout='[]') + dashboard.save() + return dashboard.to_dict() + + +class DashboardAPI(BaseResource): + def get(self, dashboard_slug=None): + dashboard = data.models.Dashboard.objects.prefetch_related('widgets__query__latest_query_data').get(slug=dashboard_slug) + return dashboard.to_dict(with_widgets=True) + + def post(self, dashboard_id): + dashboard_properties = request.json + dashboard = data.models.Dashboard.objects.get(pk=dashboard_id) + dashboard.layout = dashboard_properties['layout'] + dashboard.name = dashboard_properties['name'] + dashboard.save() + + return dashboard.to_dict(with_widgets=True) + + def delete(self, dashboard_slug): + dashboard = data.models.Dashboard.objects.get(slug=dashboard_slug) + dashboard.is_archived = True + dashboard.save() + +api.add_resource(DashboardListAPI, '/api/dashboards', endpoint='dashboards') +api.add_resource(DashboardAPI, '/api/dashboards/', endpoint='dashboard') + + +class WidgetListAPI(BaseResource): + def post(self): + widget_properties = request.json + widget_properties['options'] = json.dumps(widget_properties['options']) + widget = data.models.Widget(**widget_properties) + widget.save() + + layout = json.loads(widget.dashboard.layout) + new_row = True + + if len(layout) == 0 or widget.width == 2: + layout.append([widget.id]) + elif len(layout[-1]) == 1: + neighbour_widget = data.models.Widget.objects.get(pk=layout[-1][0]) + if neighbour_widget.width == 1: + layout[-1].append(widget.id) + new_row = False + else: + layout.append([widget.id]) + else: + layout.append([widget.id]) + + widget.dashboard.layout = json.dumps(layout) + widget.dashboard.save() + + return {'widget': widget.to_dict(), 'layout': layout, 'new_row': new_row} + + +class WidgetAPI(BaseResource): + def delete(self, widget_id): + widget = data.models.Widget.objects.get(pk=widget_id) + # TODO: reposition existing ones + layout = json.loads(widget.dashboard.layout) + layout = map(lambda row: filter(lambda w: w != widget_id, row), layout) + layout = filter(lambda row: len(row) > 0, layout) + widget.dashboard.layout = json.dumps(layout) + widget.dashboard.save() + + widget.delete() + +api.add_resource(WidgetListAPI, '/api/widgets', endpoint='widgets') +api.add_resource(WidgetAPI, '/api/widgets/', endpoint='widget') + + +class QueryListAPI(BaseResource): + def post(self): + query_def = request.json + if 'created_at' in query_def: + query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) + + query_def.pop('latest_query_data', None) + + query_def['user'] = self.current_user + query = data.models.Query(**query_def) + query.save() + + return query.to_dict(with_result=False) + + def get(self): + return [q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()] + + +class QueryAPI(BaseResource): + def post(self, query_id): + query_def = request.json + if 'created_at' in query_def: + query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) + + query_def.pop('latest_query_data', None) + + query = data.models.Query(**query_def) + fields = query_def.keys() + fields.remove('id') + query.save(update_fields=fields) + + return query.to_dict(with_result=False) + + def get(self, query_id): + q = data.models.Query.objects.get(pk=query_id) + if q: + return q.to_dict() + else: + abort(404, message="Query not found.") + +api.add_resource(QueryListAPI, '/api/queries', endpoint='queries') +api.add_resource(QueryAPI, '/api/queries/', endpoint='query') + + +class QueryResultListAPI(BaseResource): + def post(self): + params = request.json + + if params['ttl'] == 0: + query_result = None + else: + query_result = data_manager.get_query_result(params['query'], int(params['ttl'])) + + if query_result: + return {'query_result': query_result.to_dict(parse_data=True)} + else: + job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY) + return {'job': job.to_dict()} + + +class QueryResultAPI(BaseResource): + def get(self, query_result_id): + query_result = data_manager.get_query_result_by_id(query_result_id) + if query_result: + return {'query_result': query_result.to_dict(parse_data=True)} + else: + abort(404) + + +class CsvQueryResultsAPI(BaseResource): + # TODO: bring this functionality: + #def get_current_user(self): + # user = super(CsvQueryResultsHandler, self).get_current_user() + # if not user: + # api_key = self.get_argument("api_key", None) + # query = data.models.Query.objects.get(pk=self.path_args[0]) + # + # if query.api_key and query.api_key == api_key: + # user = "API-Key=%s" % api_key + # + # return user + + def get(self, query_id, query_result_id=None): + if not query_result_id: + query = data.models.Query.objects.get(pk=query_id) + if query: + query_result_id = query.latest_query_data_id + + query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id) + if query_result: + s = cStringIO.StringIO() + + query_data = json.loads(query_result.data) + writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']]) + writer.writer = utils.UnicodeWriter(s) + writer.writeheader() + for row in query_data['rows']: + for k, v in row.iteritems(): + if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100): + row[k] = datetime.datetime.fromtimestamp(v/1000.0) + + writer.writerow(row) + + return make_response(s.getvalue(), 200, {'Content-Type': "text/csv; charset=UTF-8"}) + else: + abort(404) + +api.add_resource(CsvQueryResultsAPI, '/api/queries//results/.csv', + '/api/queries//results.csv', + endpoint='csv_query_results') +api.add_resource(QueryResultListAPI, '/api/query_results', endpoint='query_results') +api.add_resource(QueryResultAPI, '/api/query_results/', endpoint='query_result') + + +class JobAPI(BaseResource): + def get(self, job_id): + # TODO: if finished, include the query result + job = data.Job.load(data_manager.redis_connection, job_id) + return {'job': job.to_dict()} + + def delete(self, job_id): + job = data.Job.load(data_manager.redis_connection, job_id) + job.cancel() + +api.add_resource(JobAPI, '/api/jobs/', endpoint='job') + +@app.route('/') +@auth.required +def send_static(filename): + return send_from_directory(settings.STATIC_ASSETS_PATH, filename) + + +if __name__ == '__main__': + app.run(debug=True) + + + diff --git a/redash/server.py b/redash/server.py deleted file mode 100644 index edd234cee..000000000 --- a/redash/server.py +++ /dev/null @@ -1,382 +0,0 @@ -""" -Tornado based API implementation for re:dash. - -Also at the moment the Tornado server is used to serve the static assets (and the Angular.js app), -but this is only due to configuration issues and temporary. - -Usage: - python server.py [--port=8888] [--debug] [--static=..] - - port - port to listen to - debug - enable debug mode (extensive logging, restart on code change) - static - static assets path - -If static option isn't specified it will be taken from settings.py. -""" -import csv -import hashlib -import json -import numbers -import os -import urlparse -import logging -import cStringIO -import datetime -import dateutil.parser -import redis -import sqlparse -import tornado.ioloop -import tornado.web -import tornado.auth -import tornado.options -import settings -import time -from data import utils -import data - - -class BaseHandler(tornado.web.RequestHandler): - def initialize(self): - self.data_manager = self.application.settings.get('data_manager', None) - self.redis_connection = self.application.settings['redis_connection'] - - def get_current_user(self): - user = self.get_secure_cookie("user") - return user - - def write_json(self, response, encode=True): - if encode: - response = json.dumps(response, cls=utils.JSONEncoder) - self.set_header("Content-Type", "application/json; charset=UTF-8") - self.write(response) - - -class BaseAuthenticatedHandler(BaseHandler): - @tornado.web.authenticated - def prepare(self): - pass - - -class PingHandler(tornado.web.RequestHandler): - def get(self): - self.write("PONG") - - -class GoogleLoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleMixin): - @tornado.web.asynchronous - @tornado.gen.coroutine - def get(self): - if self.get_argument("openid.mode", None): - user = yield self.get_authenticated_user() - - if user['email'] in settings.ALLOWED_USERS or user['email'].endswith("@%s" % settings.GOOGLE_APPS_DOMAIN): - logging.info("Authenticated: %s", user['email']) - self.set_secure_cookie("user", user['email']) - self.redirect("/") - else: - logging.error("Failed logging in with: %s", user) - self.authenticate_redirect() - else: - self.authenticate_redirect() - - -class MainHandler(BaseAuthenticatedHandler): - def get(self, *args): - email_md5 = hashlib.md5(self.current_user.lower()).hexdigest() - gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5 - - user = { - 'gravatar_url': gravatar_url, - 'is_admin': self.current_user in settings.ADMINS, - 'name': self.current_user - } - - self.render("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) - - -class QueryFormatHandler(BaseAuthenticatedHandler): - def post(self): - arguments = json.loads(self.request.body) - query = arguments.get("query", "") - - self.write(sqlparse.format(query, reindent=True, keyword_case='upper')) - - -class StatusHandler(BaseAuthenticatedHandler): - def get(self): - status = {} - info = self.redis_connection.info() - status['redis_used_memory'] = info['used_memory_human'] - - status['queries_count'] = data.models.Query.objects.count() - status['query_results_count'] = data.models.QueryResult.objects.count() - status['dashboards_count'] = data.models.Dashboard.objects.count() - status['widgets_count'] = data.models.Widget.objects.count() - - status['workers'] = [self.redis_connection.hgetall(w) - for w in self.redis_connection.smembers('workers')] - - manager_status = self.redis_connection.hgetall('manager:status') - status['manager'] = manager_status - status['manager']['queue_size'] = self.redis_connection.zcard('jobs') - - self.write_json(status) - - -class WidgetsHandler(BaseAuthenticatedHandler): - def post(self, widget_id=None): - widget_properties = json.loads(self.request.body) - widget_properties['options'] = json.dumps(widget_properties['options']) - widget = data.models.Widget(**widget_properties) - widget.save() - - layout = json.loads(widget.dashboard.layout) - new_row = True - - if len(layout) == 0 or widget.width == 2: - layout.append([widget.id]) - elif len(layout[-1]) == 1: - neighbour_widget = data.models.Widget.objects.get(pk=layout[-1][0]) - if neighbour_widget.width == 1: - layout[-1].append(widget.id) - new_row = False - else: - layout.append([widget.id]) - else: - layout.append([widget.id]) - - widget.dashboard.layout = json.dumps(layout) - widget.dashboard.save() - - self.write_json({'widget': widget.to_dict(), 'layout': layout, 'new_row': new_row}) - - def delete(self, widget_id): - widget_id = int(widget_id) - widget = data.models.Widget.objects.get(pk=widget_id) - # TODO: reposition existing ones - layout = json.loads(widget.dashboard.layout) - layout = map(lambda row: filter(lambda w: w != widget_id, row), layout) - layout = filter(lambda row: len(row) > 0, layout) - widget.dashboard.layout = json.dumps(layout) - widget.dashboard.save() - - widget.delete() - - -class DashboardHandler(BaseAuthenticatedHandler): - def get(self, dashboard_slug=None): - if dashboard_slug: - dashboard = data.models.Dashboard.objects.prefetch_related('widgets__visualization__query__latest_query_data').get(slug=dashboard_slug) - self.write_json(dashboard.to_dict(with_widgets=True)) - else: - dashboards = [d.to_dict() for d in - data.models.Dashboard.objects.filter(is_archived=False)] - self.write_json(dashboards) - - def post(self, dashboard_id): - if dashboard_id: - dashboard_properties = json.loads(self.request.body) - dashboard = data.models.Dashboard.objects.get(pk=dashboard_id) - dashboard.layout = dashboard_properties['layout'] - dashboard.name = dashboard_properties['name'] - dashboard.save() - - self.write_json(dashboard.to_dict(with_widgets=True)) - else: - dashboard_properties = json.loads(self.request.body) - dashboard = data.models.Dashboard(name=dashboard_properties['name'], - user=self.current_user, - layout='[]') - dashboard.save() - self.write_json(dashboard.to_dict()) - - def delete(self, dashboard_slug): - dashboard = data.models.Dashboard.objects.get(slug=dashboard_slug) - dashboard.is_archived = True - dashboard.save() - - -class QueriesHandler(BaseAuthenticatedHandler): - def post(self, id=None): - query_def = json.loads(self.request.body) - if 'created_at' in query_def: - query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) - - query_def.pop('latest_query_data', None) - query_def.pop('visualizations', None) - - if id: - query = data.models.Query(**query_def) - fields = query_def.keys() - fields.remove('id') - query.save(update_fields=fields) - else: - query_def['user'] = self.current_user - query = data.models.Query(**query_def) - query.save() - query.create_default_visualizations() - - self.write_json(query.to_dict(with_result=False)) - - def get(self, id=None): - if id: - q = data.models.Query.objects.get(pk=id) - if q: - self.write_json(q.to_dict(with_visualizations=True)) - else: - self.send_error(404) - else: - self.write_json([q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()]) - - -class QueryResultsHandler(BaseAuthenticatedHandler): - def get(self, query_result_id): - query_result = self.data_manager.get_query_result_by_id(query_result_id) - if query_result: - self.write_json({'query_result': query_result.to_dict(parse_data=True)}) - else: - self.send_error(404) - - def post(self, _): - params = json.loads(self.request.body) - - if params['ttl'] == 0: - query_result = None - else: - query_result = self.data_manager.get_query_result(params['query'], int(params['ttl'])) - - if query_result: - self.write_json({'query_result': query_result.to_dict(parse_data=True)}) - else: - job = self.data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY) - self.write({'job': job.to_dict()}) - - -class VisualizationHandler(BaseAuthenticatedHandler): - def get(self, id): - pass - - def post(self, id=None): - kwargs = json.loads(self.request.body) - kwargs['options'] = json.dumps(kwargs['options']) - - if id: - vis = data.models.Visualization(**kwargs) - fields = kwargs.keys() - fields.remove('id') - vis.save(update_fields=fields) - else: - vis = data.models.Visualization(**kwargs) - vis.save() - - self.write_json(vis.to_dict(with_query=False)) - - def delete(self, id): - vis = data.models.Visualization.objects.get(pk=id) - vis.delete() - - -class CsvQueryResultsHandler(BaseAuthenticatedHandler): - def get_current_user(self): - user = super(CsvQueryResultsHandler, self).get_current_user() - if not user: - api_key = self.get_argument("api_key", None) - query = data.models.Query.objects.get(pk=self.path_args[0]) - - if query.api_key and query.api_key == api_key: - user = "API-Key=%s" % api_key - - return user - - def get(self, query_id, result_id=None): - if not result_id: - query = data.models.Query.objects.get(pk=query_id) - if query: - result_id = query.latest_query_data_id - - query_result = result_id and self.data_manager.get_query_result_by_id(result_id) - if query_result: - self.set_header("Content-Type", "text/csv; charset=UTF-8") - s = cStringIO.StringIO() - - query_data = json.loads(query_result.data) - writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']]) - writer.writer = utils.UnicodeWriter(s) - writer.writeheader() - for row in query_data['rows']: - for k, v in row.iteritems(): - if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100): - row[k] = datetime.datetime.fromtimestamp(v/1000.0) - - writer.writerow(row) - - self.write(s.getvalue()) - else: - self.send_error(404) - - -class JobsHandler(BaseAuthenticatedHandler): - def get(self, job_id=None): - if job_id: - # TODO: if finished, include the query result - job = data.Job.load(self.data_manager.redis_connection, job_id) - self.write({'job': job.to_dict()}) - else: - raise NotImplemented - - def delete(self, job_id): - job = data.Job.load(self.data_manager.redis_connection, job_id) - job.cancel() - - -def get_application(static_path, is_debug, redis_connection, data_manager): - return tornado.web.Application([(r"/", MainHandler), - (r"/ping", PingHandler), - (r"/api/queries/([0-9]*)/results(?:/([0-9]*))?.csv", CsvQueryResultsHandler), - (r"/api/queries/format", QueryFormatHandler), - (r"/api/queries(?:/([0-9]*))?", QueriesHandler), - (r"/api/query_results(?:/([0-9]*))?", QueryResultsHandler), - (r"/api/jobs/(.*)", JobsHandler), - (r"/api/visualizations(?:/([0-9]*))?", VisualizationHandler), - (r"/api/widgets(?:/([0-9]*))?", WidgetsHandler), - (r"/api/dashboards(?:/(.*))?", DashboardHandler), - (r"/admin/(.*)", MainHandler), - (r"/dashboard/(.*)", MainHandler), - (r"/queries(.*)", MainHandler), - (r"/login", GoogleLoginHandler), - (r"/status.json", StatusHandler), - (r"/(.*)", tornado.web.StaticFileHandler, - {"path": static_path})], - template_path=static_path, - static_path=static_path, - debug=is_debug, - login_url="/login", - cookie_secret=settings.COOKIE_SECRET, - redis_connection=redis_connection, - data_manager=data_manager) - - -if __name__ == '__main__': - tornado.options.define("port", default=8888, type=int) - tornado.options.define("debug", default=False, type=bool) - tornado.options.define("static", default=settings.STATIC_ASSETS_PATH, type=str) - - tornado.options.parse_command_line() - - root_path = os.path.dirname(__file__) - static_path = os.path.abspath(os.path.join(root_path, tornado.options.options.static)) - - url = urlparse.urlparse(settings.REDIS_URL) - redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password) - data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, - settings.MAX_CONNECTIONS) - - logging.info("re:dash web server stating on port: %d...", tornado.options.options.port) - logging.info("UI assets path: %s...", static_path) - - application = get_application(static_path, tornado.options.options.debug, - redis_connection, data_manager) - - application.listen(tornado.options.options.port) - tornado.ioloop.IOLoop.instance().start() \ No newline at end of file From 43cfdb87270f080596381870455ee52379d1dec7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:10:16 +0200 Subject: [PATCH 158/540] Cleanup the api module --- redash/__init__.py | 38 ++++++++++++++++++++++++++++++++++++++ redash/api.py | 33 +++------------------------------ 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 8b1378917..ed93692cc 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -1 +1,39 @@ +import json +import urlparse +from flask import Flask, make_response +from flask.ext.googleauth import GoogleFederated +from flask.ext.restful import Api +from werkzeug.contrib.fixers import ProxyFix +import redis +from redash import settings +from redash.data import utils + +def setup_authentication(app): + auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) + app.wsgi_app = ProxyFix(app.wsgi_app) + app.secret_key = settings.COOKIE_SECRET + auth.force_auth_on_every_request = True + + return auth + + +app = Flask(__name__, + template_folder=settings.STATIC_ASSETS_PATH, + static_folder=settings.STATIC_ASSETS_PATH, + static_path='/static') + +auth = setup_authentication(app) +api = Api(app) + + +@api.representation('application/json') +def json_representation(data, code, headers=None): + resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code) + resp.headers.extend(headers or {}) + return resp + + +redis_url = urlparse.urlparse(settings.REDIS_URL) +redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password) +data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) diff --git a/redash/api.py b/redash/api.py index 3ccde869e..a9f77ce9f 100644 --- a/redash/api.py +++ b/redash/api.py @@ -17,46 +17,19 @@ import csv import hashlib import json import numbers -import urlparse import cStringIO import datetime import dateutil -from flask import Flask, g, render_template, send_from_directory, make_response, request, jsonify -from flask_googleauth import GoogleFederated -from flask.ext.restful import Api, Resource, abort +from flask import g, render_template, send_from_directory, make_response, request, jsonify +from flask.ext.restful import Resource, abort -import redis import sqlparse import settings from data import utils import data -app = Flask(__name__, - template_folder=settings.STATIC_ASSETS_PATH, - static_folder=settings.STATIC_ASSETS_PATH, - static_path='/static') - -api = Api(app) - -@api.representation('application/json') -def json_representation(data, code, headers=None): - resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code) - resp.headers.extend(headers or {}) - return resp - - -# TODO: move this out -url = urlparse.urlparse(settings.REDIS_URL) -redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password) -data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) - -auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) -from werkzeug.contrib.fixers import ProxyFix -app.wsgi_app = ProxyFix(app.wsgi_app) -app.secret_key = settings.COOKIE_SECRET -auth.force_auth_on_every_request = True - +from redash import app, auth, api, redis_connection, data_manager @app.route('/ping', methods=['GET']) def ping(): From 6ebfa16740b0a8abe2a4913eb3ec539b1cc64127 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:14:01 +0200 Subject: [PATCH 159/540] Move manage cli into top level --- redash/cli.py => manage.py | 0 redash/requirements.txt => requirements.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename redash/cli.py => manage.py (100%) rename redash/requirements.txt => requirements.txt (100%) diff --git a/redash/cli.py b/manage.py similarity index 100% rename from redash/cli.py rename to manage.py diff --git a/redash/requirements.txt b/requirements.txt similarity index 100% rename from redash/requirements.txt rename to requirements.txt From e4c7844cae79d19b29a672524b9d0be35a404127 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:14:15 +0200 Subject: [PATCH 160/540] Remove code duplications --- manage.py | 13 ++----------- redash/api.py | 6 +++--- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/manage.py b/manage.py index 64bea6226..24d35c11e 100644 --- a/manage.py +++ b/manage.py @@ -1,7 +1,5 @@ """ -CLI to start the workers. - -TODO: move API server startup here. +CLI to manage redash. """ import atfork atfork.monkeypatch_os_fork_functions() @@ -10,11 +8,8 @@ atfork.stdlib_fixer.fix_logging_module() import argparse import logging -import urlparse -import redis import time -import settings -import data +from redash import settings, data, redis_connection, data_manager def start_workers(data_manager): @@ -48,10 +43,6 @@ if __name__ == '__main__': parser.add_argument("command") args = parser.parse_args() - url = urlparse.urlparse(settings.REDIS_URL) - redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password) - data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) - if args.command == "worker": start_workers(data_manager) else: diff --git a/redash/api.py b/redash/api.py index a9f77ce9f..a4653bfe0 100644 --- a/redash/api.py +++ b/redash/api.py @@ -25,9 +25,9 @@ from flask import g, render_template, send_from_directory, make_response, reques from flask.ext.restful import Resource, abort import sqlparse -import settings -from data import utils -import data +from redash import settings +from redash.data import utils +from redash import data from redash import app, auth, api, redis_connection, data_manager From 806f57c627155bd395634d3f17468e9b1482f9d9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:14:54 +0200 Subject: [PATCH 161/540] Update requirements.txt. --- requirements.txt | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index dd75d2140..e70099ca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,22 @@ -psycopg2==2.5.1 -redis==2.7.5 -tornado==3.0.2 -sqlparse==0.1.8 Django==1.5.4 +Flask==0.10.1 +Flask-GoogleAuth==0.4 +Flask-RESTful==0.2.10 +Jinja2==2.7.2 +MarkupSafe==0.18 +Werkzeug==0.9.4 +aniso8601==0.82 +atfork==0.1.2 +blinker==1.3 django-db-pool==0.0.10 -qr==0.6.0 +itsdangerous==0.23 +psycopg2==2.5.1 python-dateutil==2.1 +pytz==2013.9 +qr==0.6.0 +redis==2.7.5 +requests==2.2.0 setproctitle==1.1.8 -atfork==0.1.2 \ No newline at end of file +six==1.5.2 +sqlparse==0.1.8 +wsgiref==0.1.2 From 69f7c3417e743cd3aaafdc2fcb434db2d16a8d94 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:23:06 +0200 Subject: [PATCH 162/540] Add server starting option to manage.py. --- manage.py | 23 +++++++++++++++++++---- redash/api.py | 11 +---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manage.py b/manage.py index 24d35c11e..2b42e6851 100644 --- a/manage.py +++ b/manage.py @@ -9,10 +9,10 @@ atfork.stdlib_fixer.fix_logging_module() import argparse import logging import time -from redash import settings, data, redis_connection, data_manager +from redash import settings, app, data_manager -def start_workers(data_manager): +def start_workers(): try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') @@ -34,17 +34,32 @@ def start_workers(data_manager): data_manager.stop_workers() +def start_server(port, debug): + app.run(debug=debug, port=port) + if __name__ == '__main__': channel = logging.StreamHandler() logging.getLogger().addHandler(channel) logging.getLogger().setLevel(settings.LOG_LEVEL) parser = argparse.ArgumentParser() - parser.add_argument("command") + subparsers = parser.add_subparsers(title='command', dest='command') + + subparsers.add_parser('worker', help='start query execution workers') + server_parser = subparsers.add_parser('server', help='start api server') + server_parser.add_arguemnt('--debug', + action='store_true', + help='start in debug mode (code reload)') + server_parser.add_arguemnt('--port', + default=8888, + help='port to bind to') + args = parser.parse_args() if args.command == "worker": - start_workers(data_manager) + start_workers() + elif args.command == 'server': + start_server(args.port, args.debug) else: print "Unknown command" diff --git a/redash/api.py b/redash/api.py index a4653bfe0..e828ea2be 100644 --- a/redash/api.py +++ b/redash/api.py @@ -1,17 +1,8 @@ """ Flask-restful based API implementation for re:dash. -Also at the moment the Flask server is used to serve the static assets (and the Angular.js app), +Currently the Flask server is used to serve the static assets (and the Angular.js app), but this is only due to configuration issues and temporary. - -Usage: - python api.py [--port=8888] [--debug] [--static=..] - - port - port to listen to - debug - enable debug mode (extensive logging, restart on code change) - static - static assets path - -If static option isn't specified it will be taken from settings.py. """ import csv import hashlib From 065324d256b26ce7bba994a995e11ff3581a0fb6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:28:34 +0200 Subject: [PATCH 163/540] Fix import issues (renamed api to controllers). --- manage.py | 4 ++-- redash/__init__.py | 2 ++ redash/{api.py => controllers.py} | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) rename redash/{api.py => controllers.py} (99%) diff --git a/manage.py b/manage.py index 2b42e6851..c8eeefdf9 100644 --- a/manage.py +++ b/manage.py @@ -47,10 +47,10 @@ if __name__ == '__main__': subparsers.add_parser('worker', help='start query execution workers') server_parser = subparsers.add_parser('server', help='start api server') - server_parser.add_arguemnt('--debug', + server_parser.add_argument('--debug', action='store_true', help='start in debug mode (code reload)') - server_parser.add_arguemnt('--port', + server_parser.add_argument('--port', default=8888, help='port to bind to') diff --git a/redash/__init__.py b/redash/__init__.py index ed93692cc..3e21fec44 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -37,3 +37,5 @@ def json_representation(data, code, headers=None): redis_url = urlparse.urlparse(settings.REDIS_URL) redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password) data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) + +from redash import controllers \ No newline at end of file diff --git a/redash/api.py b/redash/controllers.py similarity index 99% rename from redash/api.py rename to redash/controllers.py index e828ea2be..90e9260b0 100644 --- a/redash/api.py +++ b/redash/controllers.py @@ -22,6 +22,7 @@ from redash import data from redash import app, auth, api, redis_connection, data_manager + @app.route('/ping', methods=['GET']) def ping(): return 'PONG.' From 6d1ff98bda78b9e7dec4b4397d6f56273c58ee58 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 17 Jan 2014 11:36:44 +0200 Subject: [PATCH 164/540] Make manage.py executable. --- manage.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 manage.py diff --git a/manage.py b/manage.py old mode 100644 new mode 100755 index c8eeefdf9..719f34b3c --- a/manage.py +++ b/manage.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ CLI to manage redash. """ From 0a0f7d73659666973a9a29d458aea0fdf7989da0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 19 Jan 2014 21:59:56 +0200 Subject: [PATCH 165/540] Add API authentication support --- redash/__init__.py | 13 +--------- redash/authentication.py | 52 ++++++++++++++++++++++++++++++++++++++++ redash/controllers.py | 2 +- 3 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 redash/authentication.py diff --git a/redash/__init__.py b/redash/__init__.py index 3e21fec44..4346246ba 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -1,22 +1,11 @@ import json import urlparse from flask import Flask, make_response -from flask.ext.googleauth import GoogleFederated from flask.ext.restful import Api -from werkzeug.contrib.fixers import ProxyFix import redis from redash import settings from redash.data import utils - - -def setup_authentication(app): - auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) - app.wsgi_app = ProxyFix(app.wsgi_app) - app.secret_key = settings.COOKIE_SECRET - auth.force_auth_on_every_request = True - - return auth - +from redash.authentication import setup_authentication app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, diff --git a/redash/authentication.py b/redash/authentication.py new file mode 100644 index 000000000..bb428591a --- /dev/null +++ b/redash/authentication.py @@ -0,0 +1,52 @@ +import functools +from flask import request, make_response +from flask.ext.googleauth import GoogleFederated +from werkzeug.contrib.fixers import ProxyFix +import werkzeug.wrappers +from redash import data, settings + + +class HMACAuthentication(object): + def __init__(self, auth): + self.auth = auth + + #user = super(CsvQueryResultsHandler, self).get_current_user() + #if not user: + # api_key = self.get_argument("api_key", None) + # query = data.models.Query.objects.get(pk=self.path_args[0]) + # + # if query.api_key and query.api_key == api_key: + # user = "API-Key=%s" % api_key + # + #return user + + def required(self, fn): + wrapped_fn = self.auth.required(fn) + + @functools.wraps(fn) + def decorated(*args, **kwargs): + api_key = request.args.get('api_key') + query_id = request.view_args.get('query_id', None) + + if api_key and query_id: + query = data.models.Query.objects.get(pk=query_id) + if query.api_key and query.api_key == api_key: + return fn(*args, **kwargs) + + # Work around for flask-restful testing only for flask.wrappers.Resource instead of + # werkzeug.wrappers.Response + resp = wrapped_fn(*args, **kwargs) + if isinstance(resp, werkzeug.wrappers.Response): + resp = make_response(resp) + + return resp + + return decorated + +def setup_authentication(app): + openid_auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) + app.wsgi_app = ProxyFix(app.wsgi_app) + app.secret_key = settings.COOKIE_SECRET + openid_auth.force_auth_on_every_request = True + + return HMACAuthentication(openid_auth) diff --git a/redash/controllers.py b/redash/controllers.py index 90e9260b0..bfb0b5ccc 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -10,7 +10,7 @@ import json import numbers import cStringIO import datetime -import dateutil +import dateutil.parser from flask import g, render_template, send_from_directory, make_response, request, jsonify from flask.ext.restful import Resource, abort From b0f75678ee8cac8c943116f6d84ff6570d293b11 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jan 2014 08:55:33 +0200 Subject: [PATCH 166/540] Real HAMC authentication --- redash/authentication.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index bb428591a..56d561fcd 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -1,6 +1,9 @@ import functools +import hashlib +import hmac from flask import request, make_response from flask.ext.googleauth import GoogleFederated +import time from werkzeug.contrib.fixers import ProxyFix import werkzeug.wrappers from redash import data, settings @@ -10,27 +13,23 @@ class HMACAuthentication(object): def __init__(self, auth): self.auth = auth - #user = super(CsvQueryResultsHandler, self).get_current_user() - #if not user: - # api_key = self.get_argument("api_key", None) - # query = data.models.Query.objects.get(pk=self.path_args[0]) - # - # if query.api_key and query.api_key == api_key: - # user = "API-Key=%s" % api_key - # - #return user - def required(self, fn): wrapped_fn = self.auth.required(fn) @functools.wraps(fn) def decorated(*args, **kwargs): - api_key = request.args.get('api_key') + signature = request.args.get('signature') + expires = int(request.args.get('expires') or 0) query_id = request.view_args.get('query_id', None) - if api_key and query_id: + if signature and query_id and time.time() < expires: query = data.models.Query.objects.get(pk=query_id) - if query.api_key and query.api_key == api_key: + h = hmac.new(str(query.api_key), msg=request.path, digestmod=hashlib.sha1) + h.update(str(expires)) + + print h.hexdigest() + + if query.api_key and signature == h.hexdigest(): return fn(*args, **kwargs) # Work around for flask-restful testing only for flask.wrappers.Resource instead of From 349bfa9139b6d647380276d4452a92a017433ed1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 16:11:27 +0200 Subject: [PATCH 167/540] Remove debug print. --- redash/authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index 56d561fcd..44d373676 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -27,8 +27,6 @@ class HMACAuthentication(object): h = hmac.new(str(query.api_key), msg=request.path, digestmod=hashlib.sha1) h.update(str(expires)) - print h.hexdigest() - if query.api_key and signature == h.hexdigest(): return fn(*args, **kwargs) @@ -42,6 +40,7 @@ class HMACAuthentication(object): return decorated + def setup_authentication(app): openid_auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) app.wsgi_app = ProxyFix(app.wsgi_app) From 6eddaeda61e1c6def03fad846bbe61f5f3077e6c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 16:12:32 +0200 Subject: [PATCH 168/540] This version of GoogleAuth has no force_auth_on_every_request option. --- redash/authentication.py | 1 - 1 file changed, 1 deletion(-) diff --git a/redash/authentication.py b/redash/authentication.py index 44d373676..36f63d272 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -45,6 +45,5 @@ def setup_authentication(app): openid_auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = settings.COOKIE_SECRET - openid_auth.force_auth_on_every_request = True return HMACAuthentication(openid_auth) From 693b25efc5ee84fd6bdafae1aa30167d26f03131 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 16:14:40 +0200 Subject: [PATCH 169/540] Remove commented out code. --- redash/controllers.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index bfb0b5ccc..1792fdecf 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -238,18 +238,6 @@ class QueryResultAPI(BaseResource): class CsvQueryResultsAPI(BaseResource): - # TODO: bring this functionality: - #def get_current_user(self): - # user = super(CsvQueryResultsHandler, self).get_current_user() - # if not user: - # api_key = self.get_argument("api_key", None) - # query = data.models.Query.objects.get(pk=self.path_args[0]) - # - # if query.api_key and query.api_key == api_key: - # user = "API-Key=%s" % api_key - # - # return user - def get(self, query_id, query_result_id=None): if not query_result_id: query = data.models.Query.objects.get(pk=query_id) From cb5968bc5f2b8bfe62e96932f4fff55d43a0ce58 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 16:44:19 +0200 Subject: [PATCH 170/540] Cleanup manage.py. --- manage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/manage.py b/manage.py index 719f34b3c..cc92a9c13 100755 --- a/manage.py +++ b/manage.py @@ -38,6 +38,7 @@ def start_workers(): def start_server(port, debug): app.run(debug=debug, port=port) + if __name__ == '__main__': channel = logging.StreamHandler() logging.getLogger().addHandler(channel) @@ -60,7 +61,4 @@ if __name__ == '__main__': if args.command == "worker": start_workers() elif args.command == 'server': - start_server(args.port, args.debug) - else: - print "Unknown command" - + start_server(args.port, args.debug) \ No newline at end of file From c6c639f16fe66a4be2f2e3c39a95c6bff5f9b8a7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 18:50:15 +0200 Subject: [PATCH 171/540] Add .coverage file to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 32e64b015..bd80a38c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea *.pyc +.coverage redash/settings.py rd_ui/dist .DS_Store From cb7fbc16b0e8cb06762fa76b502d2b82bfe99269 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 18:50:25 +0200 Subject: [PATCH 172/540] Initial version of tests. --- tests/__init__.py | 1 + tests/controllers.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/controllers.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..927ea298b --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +from tests import controllers \ No newline at end of file diff --git a/tests/controllers.py b/tests/controllers.py new file mode 100644 index 000000000..4fcf0864d --- /dev/null +++ b/tests/controllers.py @@ -0,0 +1,69 @@ +from contextlib import contextmanager +import unittest +from redash import app + + +@contextmanager +def authenticated_user(c): + with c.session_transaction() as sess: + sess['openid'] = {'email': 'test@example.com', 'name': 'John Test'} + + yield + + +class AuthenticationTestMixin(): + def test_redirects_when_not_authenticated(self): + with app.test_client() as c: + for path in self.paths: + rv = c.get(path) + self.assertEquals(302, rv.status_code) + + def test_returns_content_when_authenticated(self): + with app.test_client() as c: + with authenticated_user(c): + for path in self.paths: + rv = c.get(path) + self.assertEquals(200, rv.status_code) + + +class PingTest(unittest.TestCase): + def test_ping(self): + with app.test_client() as c: + rv = c.get('/ping') + self.assertEquals(200, rv.status_code) + self.assertEquals('PONG.', rv.data) + + +class IndexTest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = ['/', '/dashboard/example', '/queries/1', '/admin/status'] + + +class StatusTest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = ['/status.json'] + + +class DashboardAPITest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = ['/api/dashboards'] + + +class QueryAPITest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = ['/api/queries'] + + +class QueryResultAPITest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = [] + + +class JobAPITest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = [] + + +class CsvQueryResultAPITest(unittest.TestCase, AuthenticationTestMixin): + def setUp(self): + self.paths = [] \ No newline at end of file From db91ca82c13ce780ead73d101b5490787668af60 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 19:52:55 +0200 Subject: [PATCH 173/540] Coverage & coveralls.io support --- .coveragerc | 4 ++++ .gitignore | 1 + 2 files changed, 5 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..812fc3b13 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[report] +omit = + */python?.?/* + */site-packages/nose/* diff --git a/.gitignore b/.gitignore index bd80a38c1..78e0bae52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.coveralls.yml .idea *.pyc .coverage From 6582bce0d38daae6de5336a77c055a5ce591116b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 19:58:39 +0200 Subject: [PATCH 174/540] Exclude settings.py from coverage report. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 812fc3b13..c10609368 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] omit = + */settings.py */python?.?/* */site-packages/nose/* From 261ecfcb118528c961921aecda61daed40b233d6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2014 19:59:01 +0200 Subject: [PATCH 175/540] make test command --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index dc8ed84e6..19c29168d 100644 --- a/Makefile +++ b/Makefile @@ -14,3 +14,6 @@ pack: upload: python bin/upload_version.py $(FULL_VERSION) $(FILENAME) + +test: + nosetests --with-coverage --cover-package=redash tests/*.py From 5198cc17d3b09994037fbf7ab9075f4f02b39929 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 12:08:49 +0200 Subject: [PATCH 176/540] peewee based models --- redash/__init__.py | 15 +++- redash/authentication.py | 4 +- redash/controllers.py | 44 +++++----- redash/models.py | 183 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 24 deletions(-) create mode 100644 redash/models.py diff --git a/redash/__init__.py b/redash/__init__.py index 4346246ba..3a5d653d5 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -2,19 +2,30 @@ import json import urlparse from flask import Flask, make_response from flask.ext.restful import Api +from flask_peewee.db import Database + import redis from redash import settings from redash.data import utils -from redash.authentication import setup_authentication + app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, static_folder=settings.STATIC_ASSETS_PATH, static_path='/static') -auth = setup_authentication(app) + api = Api(app) +# configure our database +app.config['DATABASE'] = { + 'name': 'postgres', + 'engine': 'peewee.PostgresqlDatabase', +} +db = Database(app) + +from redash.authentication import setup_authentication +auth = setup_authentication(app) @api.representation('application/json') def json_representation(data, code, headers=None): diff --git a/redash/authentication.py b/redash/authentication.py index 36f63d272..618e079da 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -6,7 +6,7 @@ from flask.ext.googleauth import GoogleFederated import time from werkzeug.contrib.fixers import ProxyFix import werkzeug.wrappers -from redash import data, settings +from redash import models, settings class HMACAuthentication(object): @@ -23,7 +23,7 @@ class HMACAuthentication(object): query_id = request.view_args.get('query_id', None) if signature and query_id and time.time() < expires: - query = data.models.Query.objects.get(pk=query_id) + query = models.Query.get(models.Query.id == query_id) h = hmac.new(str(query.api_key), msg=request.path, digestmod=hashlib.sha1) h.update(str(expires)) diff --git a/redash/controllers.py b/redash/controllers.py index 1792fdecf..ab3bfd4bd 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -21,6 +21,7 @@ from redash.data import utils from redash import data from redash import app, auth, api, redis_connection, data_manager +from redash import models @app.route('/ping', methods=['GET']) @@ -30,6 +31,7 @@ def ping(): @app.route('/admin/') @app.route('/dashboard/') +@app.route('/queries') @app.route('/queries/') @app.route('/') @auth.required @@ -53,10 +55,10 @@ def status_api(): info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] - status['queries_count'] = data.models.Query.objects.count() - status['query_results_count'] = data.models.QueryResult.objects.count() - status['dashboards_count'] = data.models.Dashboard.objects.count() - status['widgets_count'] = data.models.Widget.objects.count() + status['queries_count'] = models.Query.select().count() + status['query_results_count'] = models.QueryResult.select().count() + status['dashboards_count'] = models.Dashboard.select().count() + status['widgets_count'] = models.Widget.select().count() status['workers'] = [redis_connection.hgetall(w) for w in redis_connection.smembers('workers')] @@ -88,27 +90,28 @@ class BaseResource(Resource): class DashboardListAPI(BaseResource): def get(self): dashboards = [d.to_dict() for d in - data.models.Dashboard.objects.filter(is_archived=False)] + models.Dashboard.select().where(models.Dashboard.is_archived==False)] return dashboards def post(self): dashboard_properties = json.loads(self.request.body) - dashboard = data.models.Dashboard(name=dashboard_properties['name'], - user=self.current_user, - layout='[]') + dashboard = models.Dashboard(name=dashboard_properties['name'], + user=self.current_user, + layout='[]') dashboard.save() return dashboard.to_dict() class DashboardAPI(BaseResource): def get(self, dashboard_slug=None): - dashboard = data.models.Dashboard.objects.prefetch_related('widgets__query__latest_query_data').get(slug=dashboard_slug) + # TODO: prefetching? + dashboard = models.Dashboard.get_by_slug(dashboard_slug) return dashboard.to_dict(with_widgets=True) def post(self, dashboard_id): dashboard_properties = request.json - dashboard = data.models.Dashboard.objects.get(pk=dashboard_id) + dashboard = models.Dashboard.get(models.Dashboard.id == dashboard_id) dashboard.layout = dashboard_properties['layout'] dashboard.name = dashboard_properties['name'] dashboard.save() @@ -116,7 +119,7 @@ class DashboardAPI(BaseResource): return dashboard.to_dict(with_widgets=True) def delete(self, dashboard_slug): - dashboard = data.models.Dashboard.objects.get(slug=dashboard_slug) + dashboard = models.Dashboard.get_by_slug(dashboard_slug) dashboard.is_archived = True dashboard.save() @@ -128,7 +131,7 @@ class WidgetListAPI(BaseResource): def post(self): widget_properties = request.json widget_properties['options'] = json.dumps(widget_properties['options']) - widget = data.models.Widget(**widget_properties) + widget = models.Widget(**widget_properties) widget.save() layout = json.loads(widget.dashboard.layout) @@ -137,7 +140,7 @@ class WidgetListAPI(BaseResource): if len(layout) == 0 or widget.width == 2: layout.append([widget.id]) elif len(layout[-1]) == 1: - neighbour_widget = data.models.Widget.objects.get(pk=layout[-1][0]) + neighbour_widget = models.Widget.get(models.Widget.id == layout[-1][0]) if neighbour_widget.width == 1: layout[-1].append(widget.id) new_row = False @@ -154,7 +157,7 @@ class WidgetListAPI(BaseResource): class WidgetAPI(BaseResource): def delete(self, widget_id): - widget = data.models.Widget.objects.get(pk=widget_id) + widget = models.Widget.get(models.Widget.id == widget_id) # TODO: reposition existing ones layout = json.loads(widget.dashboard.layout) layout = map(lambda row: filter(lambda w: w != widget_id, row), layout) @@ -177,13 +180,13 @@ class QueryListAPI(BaseResource): query_def.pop('latest_query_data', None) query_def['user'] = self.current_user - query = data.models.Query(**query_def) + query = models.Query(**query_def) query.save() return query.to_dict(with_result=False) def get(self): - return [q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()] + return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()] class QueryAPI(BaseResource): @@ -194,15 +197,16 @@ class QueryAPI(BaseResource): query_def.pop('latest_query_data', None) - query = data.models.Query(**query_def) + query = models.Query(**query_def) fields = query_def.keys() fields.remove('id') - query.save(update_fields=fields) + # model.save(only=model.dirty_fields) + query.save(only=fields) return query.to_dict(with_result=False) def get(self, query_id): - q = data.models.Query.objects.get(pk=query_id) + q = models.Query.get(models.Query.id == query_id) if q: return q.to_dict() else: @@ -240,7 +244,7 @@ class QueryResultAPI(BaseResource): class CsvQueryResultsAPI(BaseResource): def get(self, query_id, query_result_id=None): if not query_result_id: - query = data.models.Query.objects.get(pk=query_id) + query = models.Query.get(models.Query.id == query_id) if query: query_result_id = query.latest_query_data_id diff --git a/redash/models.py b/redash/models.py new file mode 100644 index 000000000..797694a40 --- /dev/null +++ b/redash/models.py @@ -0,0 +1,183 @@ +import json +import hashlib +import time +import datetime +from flask.ext.peewee.utils import slugify +from redash.data import utils +import peewee +from redash import db + + +#class User(db.Model): +# id = db.Column(db.Integer, primary_key=True) +# name = db.Column(db.String(320)) +# email = db.Column(db.String(160), unique=True) +# +# def __repr__(self): +# return '' % (self.name, self.email) + + +class QueryResult(db.Model): + id = peewee.PrimaryKeyField() + query_hash = peewee.CharField(max_length=32) + query = peewee.TextField() + data = peewee.TextField() + runtime = peewee.FloatField() + retrieved_at = peewee.DateTimeField() + + class Meta: + db_table = 'query_results' + + def to_dict(self): + return { + 'id': self.id, + 'query_hash': self.query_hash, + 'query': self.query, + 'data': json.loads(self.data), + 'runtime': self.runtime, + 'retrieved_at': self.retrieved_at + } + + def __unicode__(self): + return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) + + +class Query(db.Model): + id = peewee.PrimaryKeyField() + latest_query_data = peewee.ForeignKeyField(QueryResult, null=True) + #latest_query_data_id = peewee.IntegerField() + name = peewee.CharField(max_length=255) + description = peewee.CharField(max_length=4096) + query = peewee.TextField() + query_hash = peewee.CharField(max_length=32) + api_key = peewee.CharField(max_length=40) + ttl = peewee.IntegerField() + user = peewee.CharField(max_length=360) + created_at = peewee.DateTimeField(default=datetime.datetime.now) + + class Meta: + db_table = 'queries' + + def to_dict(self, with_result=True, with_stats=False): + d = { + 'id': self.id, + 'latest_query_data_id': self._data['latest_query_data'], + 'name': self.name, + 'description': self.description, + 'query': self.query, + 'query_hash': self.query_hash, + 'ttl': self.ttl, + 'user': self.user, + 'api_key': self.api_key, + 'created_at': self.created_at, + } + + if with_stats: + d['avg_runtime'] = self.avg_runtime + d['min_runtime'] = self.min_runtime + d['max_runtime'] = self.max_runtime + d['last_retrieved_at'] = self.last_retrieved_at + d['times_retrieved'] = self.times_retrieved + + if with_result and self._data['latest_query_data']: + d['latest_query_data'] = self.latest_query_data.to_dict() + + return d + + @classmethod + def all_queries(cls): + query = """SELECT queries.*, query_stats.* +FROM queries +LEFT OUTER JOIN + (SELECT qu.query_hash, + count(0) AS "times_retrieved", + avg(runtime) AS "avg_runtime", + min(runtime) AS "min_runtime", + max(runtime) AS "max_runtime", + max(retrieved_at) AS "last_retrieved_at" + FROM queries qu + JOIN query_results qr ON qu.query_hash=qr.query_hash + GROUP BY qu.query_hash) query_stats ON query_stats.query_hash = queries.query_hash + """ + return cls.raw(query) + + def save(self, *args, **kwargs): + self.query_hash = utils.gen_query_hash(self.query) + self._set_api_key() + super(Query, self).save(*args, **kwargs) + + def _set_api_key(self): + if not self.api_key: + self.api_key = hashlib.sha1( + u''.join([str(time.time()), self.query, self.user, self.name])).hexdigest() + + def __unicode__(self): + return unicode(self.id) + + +class Dashboard(db.Model): + id = peewee.PrimaryKeyField() + slug = peewee.CharField(max_length=140, index=True) + name = peewee.CharField(max_length=100) + user = peewee.CharField(max_length=360) + layout = peewee.TextField() + is_archived = peewee.BooleanField(default=False, index=True) + + class Meta: + db_table = 'dashboards' + + def to_dict(self, with_widgets=False): + layout = json.loads(self.layout) + + if with_widgets: + widgets = {w.id: w.to_dict() for w in self.widgets} + widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) + else: + widgets_layout = None + + return { + 'id': self.id, + 'slug': self.slug, + 'name': self.name, + 'user': self.user, + 'layout': layout, + 'widgets': widgets_layout + } + + @classmethod + def get_by_slug(cls, slug): + return cls.get(cls.slug==slug) + + def save(self, *args, **kwargs): + # TODO: make sure slug is unique + if not self.slug: + self.slug = slugify(self.name) + super(Dashboard, self).save(*args, **kwargs) + + def __unicode__(self): + return u"%s=%s" % (self.id, self.name) + + +class Widget(db.Model): + id = peewee.PrimaryKeyField() + query = peewee.ForeignKeyField(Query) + type = peewee.CharField(max_length=100) + width = peewee.IntegerField() + options = peewee.TextField() + dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets') + + class Meta: + db_table = 'widgets' + + def to_dict(self): + return { + 'id': self.id, + 'query': self.query.to_dict(), + 'type': self.type, + 'width': self.width, + 'options': json.loads(self.options), + 'dashboard_id': self._data['dashboard'] + } + + def __unicode__(self): + return u"%s=>%s" % (self.id, self.dashboard_id) \ No newline at end of file From fb826ec838f0fabf6c04ad0f481b8897d1e6166e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 12:09:09 +0200 Subject: [PATCH 177/540] Remove Django models --- redash/data/__init__.py | 1 - redash/data/models.py | 216 ---------------------------------------- 2 files changed, 217 deletions(-) delete mode 100644 redash/data/models.py diff --git a/redash/data/__init__.py b/redash/data/__init__.py index 56b4ed1e5..fedf2a0fb 100644 --- a/redash/data/__init__.py +++ b/redash/data/__init__.py @@ -1,4 +1,3 @@ from manager import Manager from worker import Job -import models import utils diff --git a/redash/data/models.py b/redash/data/models.py deleted file mode 100644 index c56ef3eda..000000000 --- a/redash/data/models.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -Django ORM based models to describe the data model of re:dash. -""" -import hashlib -import json -import time -from django.db import models -from django.template.defaultfilters import slugify -import utils - - -class QueryResult(models.Model): - id = models.AutoField(primary_key=True) - query_hash = models.CharField(max_length=32) - query = models.TextField() - data = models.TextField() - runtime = models.FloatField() - retrieved_at = models.DateTimeField() - - class Meta: - app_label = 'redash' - db_table = 'query_results' - - def to_dict(self): - return { - 'id': self.id, - 'query_hash': self.query_hash, - 'query': self.query, - 'data': json.loads(self.data), - 'runtime': self.runtime, - 'retrieved_at': self.retrieved_at - } - - def __unicode__(self): - return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) - - -class Query(models.Model): - id = models.AutoField(primary_key=True) - latest_query_data = models.ForeignKey(QueryResult) - name = models.CharField(max_length=255) - description = models.CharField(max_length=4096) - query = models.TextField() - query_hash = models.CharField(max_length=32) - api_key = models.CharField(max_length=40) - ttl = models.IntegerField() - user = models.CharField(max_length=360) - created_at = models.DateTimeField(auto_now_add=True) - - class Meta: - app_label = 'redash' - db_table = 'queries' - - def create_default_visualizations(self): - table_visualization = Visualization(query=self, name="Table", - description=self.description, - type="TABLE", options="{}") - table_visualization.save() - - def to_dict(self, with_result=True, with_stats=False, - with_visualizations=False): - d = { - 'id': self.id, - 'latest_query_data_id': self.latest_query_data_id, - 'name': self.name, - 'description': self.description, - 'query': self.query, - 'query_hash': self.query_hash, - 'ttl': self.ttl, - 'user': self.user, - 'api_key': self.api_key, - 'created_at': self.created_at, - } - - if with_stats: - d['avg_runtime'] = self.avg_runtime - d['min_runtime'] = self.min_runtime - d['max_runtime'] = self.max_runtime - d['last_retrieved_at'] = self.last_retrieved_at - d['times_retrieved'] = self.times_retrieved - - if with_result and self.latest_query_data_id: - d['latest_query_data'] = self.latest_query_data.to_dict() - - if with_visualizations: - d['visualizations'] = [vis.to_dict(with_query=False) - for vis in self.visualizations.all()] - - return d - - @classmethod - def all_queries(cls): - query = """SELECT queries.*, query_stats.* -FROM queries -LEFT OUTER JOIN - (SELECT qu.query_hash, - count(0) AS "times_retrieved", - avg(runtime) AS "avg_runtime", - min(runtime) AS "min_runtime", - max(runtime) AS "max_runtime", - max(retrieved_at) AS "last_retrieved_at" - FROM queries qu - JOIN query_results qr ON qu.query_hash=qr.query_hash - GROUP BY qu.query_hash) query_stats ON query_stats.query_hash = queries.query_hash - """ - return cls.objects.raw(query) - - def save(self, *args, **kwargs): - self.query_hash = utils.gen_query_hash(self.query) - self._set_api_key() - super(Query, self).save(*args, **kwargs) - - def _set_api_key(self): - if not self.api_key: - self.api_key = hashlib.sha1( - u''.join([str(time.time()), self.query, self.user, self.name])).hexdigest() - - def __unicode__(self): - return unicode(self.id) - - -class Dashboard(models.Model): - id = models.AutoField(primary_key=True) - slug = models.CharField(max_length=140) - name = models.CharField(max_length=100) - user = models.CharField(max_length=360) - layout = models.TextField() - is_archived = models.BooleanField(default=False) - - class Meta: - app_label = 'redash' - db_table = 'dashboards' - - def to_dict(self, with_widgets=False): - layout = json.loads(self.layout) - - if with_widgets: - widgets = {w.id: w.to_dict() for w in self.widgets.all()} - widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) - else: - widgets_layout = None - - return { - 'id': self.id, - 'slug': self.slug, - 'name': self.name, - 'user': self.user, - 'layout': layout, - 'widgets': widgets_layout - } - - def save(self, *args, **kwargs): - # TODO: make sure slug is unique - if not self.slug: - self.slug = slugify(self.name) - super(Dashboard, self).save(*args, **kwargs) - - def __unicode__(self): - return u"%s=%s" % (self.id, self.name) - - -class Visualization(models.Model): - id = models.AutoField(primary_key=True) - type = models.CharField(max_length=100) - query = models.ForeignKey(Query, related_name='visualizations') - name = models.CharField(max_length=255) - description = models.CharField(max_length=4096) - options = models.TextField() - - class Meta: - app_label = 'redash' - db_table = 'visualizations' - - def to_dict(self, with_query=True): - d = { - 'id': self.id, - 'type': self.type, - 'name': self.name, - 'description': self.description, - 'options': json.loads(self.options), - } - - if with_query: - d['query'] = self.query.to_dict() - - return d - - def __unicode__(self): - return u"%s=>%s" % (self.id, self.query_id) - - -class Widget(models.Model): - id = models.AutoField(primary_key=True) - type = models.CharField(max_length=100) - query = models.ForeignKey(Query, related_name='widgets') - visualization = models.ForeignKey(Visualization, related_name='widgets') - width = models.IntegerField() - options = models.TextField() - dashboard = models.ForeignKey(Dashboard, related_name='widgets') - - class Meta: - app_label = 'redash' - db_table = 'widgets' - - def to_dict(self): - return { - 'id': self.id, - 'type': self.type, - 'width': self.width, - 'options': json.loads(self.options), - 'visualization': self.visualization.to_dict(), - 'dashboard_id': self.dashboard_id - } - - def __unicode__(self): - return u"%s=>%s" % (self.id, self.dashboard_id) From 5fb910b8860537ccb75694d35cbaac0181999c2b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 12:09:42 +0200 Subject: [PATCH 178/540] Remove Django from config --- redash/settings_example.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/redash/settings_example.py b/redash/settings_example.py index a20f35f64..69e03c896 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -1,9 +1,6 @@ """ Example settings module. You should make your own copy as settings.py and enter the real settings. """ - -import django.conf - REDIS_URL = "redis://localhost:6379" # Either "pg" or "mysql" @@ -28,14 +25,3 @@ MAX_CONNECTIONS = 3 COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" LOG_LEVEL = "INFO" ANALYTICS = "" - -# Configuration of the operational database for the Django models -django.conf.settings.configure(DATABASES = { 'default': { - 'ENGINE': 'dbpool.db.backends.postgresql_psycopg2', - 'OPTIONS': {'MAX_CONNS': 10, 'MIN_CONNS': 1}, - 'NAME': 'postgres', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', - },}, TIME_ZONE = 'UTC') From 99dac8f6fdd8f633d4fc365aa91b79204f57cfc8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 12:09:58 +0200 Subject: [PATCH 179/540] Remove Django from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e70099ca1..f38c1fb41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -Django==1.5.4 Flask==0.10.1 Flask-GoogleAuth==0.4 Flask-RESTful==0.2.10 From 1586860e15df6639570638be9d1f7be2fb909acc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:22:50 +0200 Subject: [PATCH 180/540] Use peewee in data.Manager. --- redash/__init__.py | 13 ++++++------- redash/controllers.py | 3 +-- redash/data/__init__.py | 1 - redash/data/manager.py | 26 +++++++++++++------------- redash/data/query_runner.py | 8 +++++--- redash/data/worker.py | 2 +- redash/models.py | 3 +-- redash/settings_example.py | 7 +++++-- redash/{data => }/utils.py | 0 9 files changed, 32 insertions(+), 31 deletions(-) rename redash/{data => }/utils.py (100%) diff --git a/redash/__init__.py b/redash/__init__.py index 3a5d653d5..69ca6a721 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -5,8 +5,7 @@ from flask.ext.restful import Api from flask_peewee.db import Database import redis -from redash import settings -from redash.data import utils +from redash import settings, utils app = Flask(__name__, @@ -18,10 +17,8 @@ app = Flask(__name__, api = Api(app) # configure our database -app.config['DATABASE'] = { - 'name': 'postgres', - 'engine': 'peewee.PostgresqlDatabase', -} +settings.DATABASE_CONFIG.update({'threadlocals': True}) +app.config['DATABASE'] = settings.DATABASE_CONFIG db = Database(app) from redash.authentication import setup_authentication @@ -36,6 +33,8 @@ def json_representation(data, code, headers=None): redis_url = urlparse.urlparse(settings.REDIS_URL) redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password) -data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS) + +from redash import data +data_manager = data.Manager(redis_connection, db) from redash import controllers \ No newline at end of file diff --git a/redash/controllers.py b/redash/controllers.py index ab3bfd4bd..4081c1821 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -16,8 +16,7 @@ from flask import g, render_template, send_from_directory, make_response, reques from flask.ext.restful import Resource, abort import sqlparse -from redash import settings -from redash.data import utils +from redash import settings, utils from redash import data from redash import app, auth, api, redis_connection, data_manager diff --git a/redash/data/__init__.py b/redash/data/__init__.py index fedf2a0fb..0291856ee 100644 --- a/redash/data/__init__.py +++ b/redash/data/__init__.py @@ -1,3 +1,2 @@ from manager import Manager from worker import Job -import utils diff --git a/redash/data/manager.py b/redash/data/manager.py index 52d2dd54b..007e0a88e 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -1,18 +1,17 @@ """ Data manager. Used to manage and coordinate execution of queries. """ -import collections from contextlib import contextmanager +import collections import json +import time import logging import psycopg2 import qr import redis -import time -import worker -import settings -from utils import gen_query_hash - +from redash import settings +from redash.data import worker +from redash.utils import gen_query_hash class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')): def to_dict(self, parse_data=False): @@ -25,10 +24,10 @@ class QueryResult(collections.namedtuple('QueryData', 'id query data runtime ret class Manager(object): - def __init__(self, redis_connection, db_connection_string, db_max_connections): + def __init__(self, redis_connection, db): self.redis_connection = redis_connection + self.db = db self.workers = [] - self.db_connection_string = db_connection_string self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 self.status = { @@ -176,17 +175,18 @@ class Manager(object): @contextmanager def db_transaction(self): - connection = psycopg2.connect(self.db_connection_string) - cursor = connection.cursor() + self.db.connect_db() + + cursor = self.db.database.get_cursor() try: yield cursor except: - connection.rollback() + self.db.database.rollback() raise else: - connection.commit() + self.db.database.commit() finally: - connection.close() + self.db.close_db(None) def _save_status(self): self.redis_connection.hmset('manager:status', self.status) diff --git a/redash/data/query_runner.py b/redash/data/query_runner.py index 15b38a3ed..e44017e30 100644 --- a/redash/data/query_runner.py +++ b/redash/data/query_runner.py @@ -8,10 +8,12 @@ query language (for example: HiveQL). """ import logging import json -import psycopg2 import sys import select -from .utils import JSONEncoder + +import psycopg2 + +from redash.utils import JSONEncoder def redshift(connection_string): def column_friendly_name(column_name): @@ -64,4 +66,4 @@ def redshift(connection_string): return json_data, error - return query_runner \ No newline at end of file + return query_runner diff --git a/redash/data/worker.py b/redash/data/worker.py index d75ac1951..88cc92a8e 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -11,7 +11,7 @@ import time import signal import setproctitle import redis -from utils import gen_query_hash +from redash.utils import gen_query_hash class Job(object): diff --git a/redash/models.py b/redash/models.py index 797694a40..d76b61f7c 100644 --- a/redash/models.py +++ b/redash/models.py @@ -3,9 +3,8 @@ import hashlib import time import datetime from flask.ext.peewee.utils import slugify -from redash.data import utils import peewee -from redash import db +from redash import db, utils #class User(db.Model): diff --git a/redash/settings_example.py b/redash/settings_example.py index 69e03c896..bfb730911 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -9,8 +9,11 @@ CONNECTION_ADAPTER = "mysql" # -- example mysql CONNECTION_STRING = "Server=;User=;Pwd=;Database=" # -- example pg CONNECTION_STRING = "user= password= host= port=5439 dbname=" CONNECTION_STRING = "user= password= host= port=5439 dbname=" -# Connection string for the operational databases (where we store the queries, results, etc) -INTERNAL_DB_CONNECTION_STRING = "dbname=postgres" +# Connection settings for re:dash's own database (where we store the queries, results, etc) +DATABASE_CONFIG = { + 'name': 'postgres', + 'engine': 'peewee.PostgresqlDatabase', +} # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed # access GOOGLE_APPS_DOMAIN = "" diff --git a/redash/data/utils.py b/redash/utils.py similarity index 100% rename from redash/data/utils.py rename to redash/utils.py From 6ff07b99dc92189c3a1887d8d180d6c9eb8f7386 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:24:34 +0200 Subject: [PATCH 181/540] Remove MAX_CONNECTIONS from example settings --- redash/settings_example.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redash/settings_example.py b/redash/settings_example.py index bfb730911..43056090f 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -24,7 +24,6 @@ ALLOWED_USERS = [] ADMINS = [] STATIC_ASSETS_PATH = "../rd_ui/dist/" WORKERS_COUNT = 2 -MAX_CONNECTIONS = 3 COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" LOG_LEVEL = "INFO" -ANALYTICS = "" +ANALYTICS = "" \ No newline at end of file From b66d5daad06b219d7c57db8002f6c1a67778243e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:27:36 +0200 Subject: [PATCH 182/540] Set needed fields as indexed. --- redash/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/redash/models.py b/redash/models.py index d76b61f7c..9d36e7e7c 100644 --- a/redash/models.py +++ b/redash/models.py @@ -18,7 +18,7 @@ from redash import db, utils class QueryResult(db.Model): id = peewee.PrimaryKeyField() - query_hash = peewee.CharField(max_length=32) + query_hash = peewee.CharField(max_length=32, index=True) query = peewee.TextField() data = peewee.TextField() runtime = peewee.FloatField() @@ -44,7 +44,6 @@ class QueryResult(db.Model): class Query(db.Model): id = peewee.PrimaryKeyField() latest_query_data = peewee.ForeignKeyField(QueryResult, null=True) - #latest_query_data_id = peewee.IntegerField() name = peewee.CharField(max_length=255) description = peewee.CharField(max_length=4096) query = peewee.TextField() @@ -163,7 +162,7 @@ class Widget(db.Model): type = peewee.CharField(max_length=100) width = peewee.IntegerField() options = peewee.TextField() - dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets') + dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) class Meta: db_table = 'widgets' From 2aec982577bd210cf0af69c64a30780c3605cd37 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:28:54 +0200 Subject: [PATCH 183/540] Add created_at to all models. (#10) --- redash/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redash/models.py b/redash/models.py index 9d36e7e7c..86370d540 100644 --- a/redash/models.py +++ b/redash/models.py @@ -120,6 +120,7 @@ class Dashboard(db.Model): user = peewee.CharField(max_length=360) layout = peewee.TextField() is_archived = peewee.BooleanField(default=False, index=True) + created_at = peewee.DateTimeField(default=datetime.datetime.now) class Meta: db_table = 'dashboards' @@ -163,6 +164,7 @@ class Widget(db.Model): width = peewee.IntegerField() options = peewee.TextField() dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) + created_at = peewee.DateTimeField(default=datetime.datetime.now) class Meta: db_table = 'widgets' From f890e590e1a7ad195fa111f1db8887035e1a3251 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:35:06 +0200 Subject: [PATCH 184/540] Updated requirements.txt (flask-peewee). --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f38c1fb41..d423444d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,12 +3,14 @@ Flask-GoogleAuth==0.4 Flask-RESTful==0.2.10 Jinja2==2.7.2 MarkupSafe==0.18 +WTForms==1.0.5 Werkzeug==0.9.4 aniso8601==0.82 atfork==0.1.2 blinker==1.3 -django-db-pool==0.0.10 +flask-peewee==0.6.5 itsdangerous==0.23 +peewee==2.2.0 psycopg2==2.5.1 python-dateutil==2.1 pytz==2013.9 @@ -19,3 +21,4 @@ setproctitle==1.1.8 six==1.5.2 sqlparse==0.1.8 wsgiref==0.1.2 +wtf-peewee==0.2.2 From 41d00543d03b96a5c3ab1fadbbbb81237b8b49f6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 1 Feb 2014 17:44:03 +0200 Subject: [PATCH 185/540] Create db task instead of SQL tables. --- manage.py | 15 +++++++++++ redash/data/tables.sql | 56 ------------------------------------------ redash/models.py | 17 ++++++++++++- 3 files changed, 31 insertions(+), 57 deletions(-) delete mode 100644 redash/data/tables.sql diff --git a/manage.py b/manage.py index cc92a9c13..a379b0a44 100755 --- a/manage.py +++ b/manage.py @@ -39,6 +39,12 @@ def start_server(port, debug): app.run(debug=debug, port=port) +def create_db(create_tables, drop_tables): + from redash.models import create_db + + create_db(create_tables, drop_tables) + + if __name__ == '__main__': channel = logging.StreamHandler() logging.getLogger().addHandler(channel) @@ -55,10 +61,19 @@ if __name__ == '__main__': server_parser.add_argument('--port', default=8888, help='port to bind to') + db_parser = subparsers.add_parser('db', help='database operations') + db_parser.add_argument('--create-tables', + action='store_true', + help='Create all tables') + db_parser.add_argument('--drop-tables', + action='store_true', + help='Drop all tables') args = parser.parse_args() if args.command == "worker": start_workers() + elif args.command == 'db': + create_db(args.create_tables, args.drop_tables) elif args.command == 'server': start_server(args.port, args.debug) \ No newline at end of file diff --git a/redash/data/tables.sql b/redash/data/tables.sql deleted file mode 100644 index 5fb00a68d..000000000 --- a/redash/data/tables.sql +++ /dev/null @@ -1,56 +0,0 @@ -BEGIN; -CREATE TABLE "query_results" ( - "id" serial NOT NULL PRIMARY KEY, - "query_hash" varchar(32) NOT NULL, - "query" text NOT NULL, - "data" text NOT NULL, - "runtime" double precision NOT NULL, - "retrieved_at" timestamp with time zone NOT NULL -) -; -CREATE TABLE "queries" ( - "id" serial NOT NULL PRIMARY KEY, - "latest_query_data_id" integer REFERENCES "query_results" ("id") DEFERRABLE INITIALLY DEFERRED, - "name" varchar(255) NOT NULL, - "description" varchar(4096), - "query" text NOT NULL, - "query_hash" varchar(32) NOT NULL, - "api_key" varchar(40), - "ttl" integer NOT NULL, - "user" varchar(360) NOT NULL, - "created_at" timestamp with time zone NOT NULL -) -; -CREATE TABLE "dashboards" ( - "id" serial NOT NULL PRIMARY KEY, - "slug" varchar(140) NOT NULL, - "name" varchar(100) NOT NULL, - "user" varchar(360) NOT NULL, - "layout" text NOT NULL, - "is_archived" boolean NOT NULL -) -; -CREATE TABLE "visualizations" ( - "id" serial NOT NULL PRIMARY KEY, - "type" varchar(100) NOT NULL, - "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, - "name" varchar(255) NOT NULL, - "description" varchar(4096), - "options" text NOT NULL -) -; -CREATE TABLE "widgets" ( - "id" serial NOT NULL PRIMARY KEY, - "type" varchar(100) NOT NULL, - "width" integer NOT NULL, - "options" text NOT NULL, - "query_id" integer, - "visualization_id" integer NOT NULL REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED, - "dashboard_id" integer NOT NULL REFERENCES "dashboards" ("id") DEFERRABLE INITIALLY DEFERRED -) -; -CREATE INDEX "queries_latest_query_data_id" ON "queries" ("latest_query_data_id"); -CREATE INDEX "widgets_query_id" ON "widgets" ("query_id"); -CREATE INDEX "widgets_dashboard_id" ON "widgets" ("dashboard_id"); - -COMMIT; diff --git a/redash/models.py b/redash/models.py index 86370d540..c2ad79ba5 100644 --- a/redash/models.py +++ b/redash/models.py @@ -180,4 +180,19 @@ class Widget(db.Model): } def __unicode__(self): - return u"%s=>%s" % (self.id, self.dashboard_id) \ No newline at end of file + return u"%s=>%s" % (self.id, self.dashboard_id) + + +def create_db(create_tables, drop_tables): + all_models = (QueryResult, Query, Dashboard, Widget) + + db.connect_db() + + for model in all_models: + if drop_tables and model.table_exists(): + model.drop_table() + + if create_tables: + model.create_table() + + db.close_db(None) From 69ec362a8dc7bc83be87fe05f853b9893b55a9ab Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Feb 2014 09:59:50 +0200 Subject: [PATCH 186/540] Fix: make sure all dashboard slug are unique --- redash/models.py | 11 +++++++++-- tests/models.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/models.py diff --git a/redash/models.py b/redash/models.py index c2ad79ba5..d13796763 100644 --- a/redash/models.py +++ b/redash/models.py @@ -148,9 +148,14 @@ class Dashboard(db.Model): return cls.get(cls.slug==slug) def save(self, *args, **kwargs): - # TODO: make sure slug is unique if not self.slug: self.slug = slugify(self.name) + + tries = 1 + while self.select().where(Dashboard.slug == self.slug).first() is not None: + self.slug = slugify(self.name) + "_{0}".format(tries) + tries += 1 + super(Dashboard, self).save(*args, **kwargs) def __unicode__(self): @@ -190,7 +195,9 @@ def create_db(create_tables, drop_tables): for model in all_models: if drop_tables and model.table_exists(): - model.drop_table() + # TODO: submit PR to peewee to allow passing cascade option to drop_table. + db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table) + #model.drop_table() if create_tables: model.create_table() diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 000000000..c38839ed8 --- /dev/null +++ b/tests/models.py @@ -0,0 +1,31 @@ +from unittest import TestCase +from redash import settings, db, app, models + +settings.DATABASE_CONFIG = { + 'name': 'rd_test', + 'engine': 'peewee.PostgresqlDatabase', + 'threadlocals': True +} +app.config['DATABASE'] = settings.DATABASE_CONFIG +db.load_database() + + +class DatabaseTestCase(TestCase): + def setUp(self): + models.create_db(True, True) + + def tearDown(self): + models.create_db(False, True) + + +class DashboardTest(DatabaseTestCase): + def test_appends_suffix_to_slug_when_duplicate(self): + d1 = models.Dashboard.create(name='test', user='arik', layout='') + self.assertEquals(d1.slug, 'test') + + d2 = models.Dashboard.create(name='test', user='arik', layout='') + self.assertNotEquals(d1.slug, d2.slug) + + d3 = models.Dashboard.create(name='test', user='arik', layout='') + self.assertNotEquals(d1.slug, d3.slug) + self.assertNotEquals(d2.slug, d3.slug) \ No newline at end of file From ff62fbbcf4a080ba2294fea1bb72663e54779eb0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 11:01:06 +0200 Subject: [PATCH 187/540] More tests (Dasboard API). --- Makefile | 2 +- redash/controllers.py | 19 +++++---- redash/models.py | 2 +- tests/__init__.py | 57 ++++++++++++++++++++++++- tests/controllers.py | 97 +++++++++++++++++++++++++++++++++++-------- tests/models.py | 27 +++--------- 6 files changed, 154 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index 19c29168d..0fe935305 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ NAME=redash -VERSION=0.2 +VERSION=0.3 FULL_VERSION=$(VERSION).$(CIRCLE_BUILD_NUM) FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz diff --git a/redash/controllers.py b/redash/controllers.py index 4081c1821..b132120ae 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -94,7 +94,7 @@ class DashboardListAPI(BaseResource): return dashboards def post(self): - dashboard_properties = json.loads(self.request.body) + dashboard_properties = request.get_json(force=True) dashboard = models.Dashboard(name=dashboard_properties['name'], user=self.current_user, layout='[]') @@ -104,13 +104,18 @@ class DashboardListAPI(BaseResource): class DashboardAPI(BaseResource): def get(self, dashboard_slug=None): - # TODO: prefetching? - dashboard = models.Dashboard.get_by_slug(dashboard_slug) + # TODO: prefetching of widgets and queries? + try: + dashboard = models.Dashboard.get_by_slug(dashboard_slug) + except models.Dashboard.DoesNotExist: + abort(404) + return dashboard.to_dict(with_widgets=True) - def post(self, dashboard_id): - dashboard_properties = request.json - dashboard = models.Dashboard.get(models.Dashboard.id == dashboard_id) + def post(self, dashboard_slug): + # TODO: either convert all requests to use slugs or ids + dashboard_properties = request.get_json(force=True) + dashboard = models.Dashboard.get(models.Dashboard.id == dashboard_slug) dashboard.layout = dashboard_properties['layout'] dashboard.name = dashboard_properties['name'] dashboard.save() @@ -128,7 +133,7 @@ api.add_resource(DashboardAPI, '/api/dashboards/', endpoint='das class WidgetListAPI(BaseResource): def post(self): - widget_properties = request.json + widget_properties = request.get_json(force=True) widget_properties['options'] = json.dumps(widget_properties['options']) widget = models.Widget(**widget_properties) widget.save() diff --git a/redash/models.py b/redash/models.py index d13796763..d1a9d8084 100644 --- a/redash/models.py +++ b/redash/models.py @@ -202,4 +202,4 @@ def create_db(create_tables, drop_tables): if create_tables: model.create_table() - db.close_db(None) + db.close_db(None) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 927ea298b..6618283a3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,56 @@ -from tests import controllers \ No newline at end of file +from unittest import TestCase +from redash import settings, db, app +import redash.models + +# TODO: this isn't pretty... :-) +settings.DATABASE_CONFIG = { + 'name': 'rd_test', + 'engine': 'peewee.PostgresqlDatabase', + 'threadlocals': True +} +app.config['DATABASE'] = settings.DATABASE_CONFIG +db.load_database() + + +def model_factory(model, **kwargs): + def factory(**properties): + kwargs.update(properties) + + return model(**kwargs) + + return factory + + +class ModelFactory(object): + def __init__(self, model, **kwargs): + self.model = model + self.kwargs = kwargs + + def _get_kwargs(self, override_kwargs): + kwargs = self.kwargs.copy() + kwargs.update(override_kwargs) + return kwargs + + def instance(self, **override_kwargs): + kwargs = self._get_kwargs(override_kwargs) + + return self.model(**kwargs) + + def create(self, **override_kwargs): + kwargs = self._get_kwargs(override_kwargs) + + return self.model.create(**kwargs) + +DashboardFactory = ModelFactory(redash.models.Dashboard, + name='test', user='test@everything.me', layout='[]') + +dashboard_factory = model_factory(redash.models.Dashboard, name='test', user='test@everything.me', layout='[]') + + +class BaseTestCase(TestCase): + def setUp(self): + redash.models.create_db(True, True) + + def tearDown(self): + db.close_db(None) + redash.models.create_db(False, True) \ No newline at end of file diff --git a/tests/controllers.py b/tests/controllers.py index 4fcf0864d..f3c5d2686 100644 --- a/tests/controllers.py +++ b/tests/controllers.py @@ -1,16 +1,31 @@ from contextlib import contextmanager -import unittest -from redash import app +import json +from tests import BaseTestCase, DashboardFactory +from redash import app, models @contextmanager -def authenticated_user(c): +def authenticated_user(c, user='test@example.com', name='John Test'): with c.session_transaction() as sess: - sess['openid'] = {'email': 'test@example.com', 'name': 'John Test'} + sess['openid'] = {'email': user, 'name': name} yield +def json_request(method, path, data=None): + if data: + response = method(path, data=json.dumps(data)) + else: + response = method(path) + + if response.data: + response.json = json.loads(response.data) + else: + response.json = None + + return response + + class AuthenticationTestMixin(): def test_redirects_when_not_authenticated(self): with app.test_client() as c: @@ -19,14 +34,13 @@ class AuthenticationTestMixin(): self.assertEquals(302, rv.status_code) def test_returns_content_when_authenticated(self): - with app.test_client() as c: - with authenticated_user(c): - for path in self.paths: - rv = c.get(path) - self.assertEquals(200, rv.status_code) + with app.test_client() as c, authenticated_user(c): + for path in self.paths: + rv = c.get(path) + self.assertEquals(200, rv.status_code) -class PingTest(unittest.TestCase): +class PingTest(BaseTestCase): def test_ping(self): with app.test_client() as c: rv = c.get('/ping') @@ -34,36 +48,83 @@ class PingTest(unittest.TestCase): self.assertEquals('PONG.', rv.data) -class IndexTest(unittest.TestCase, AuthenticationTestMixin): +class IndexTest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = ['/', '/dashboard/example', '/queries/1', '/admin/status'] + super(IndexTest, self).setUp() -class StatusTest(unittest.TestCase, AuthenticationTestMixin): +class StatusTest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = ['/status.json'] + super(StatusTest, self).setUp() -class DashboardAPITest(unittest.TestCase, AuthenticationTestMixin): +class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = ['/api/dashboards'] + super(DashboardAPITest, self).setUp() + + def test_get_dashboard(self): + d1 = DashboardFactory.create() + with app.test_client() as c, authenticated_user(c): + rv = c.get('/api/dashboards/{0}'.format(d1.slug)) + self.assertEquals(rv.status_code, 200) + self.assertDictEqual(json.loads(rv.data), d1.to_dict(with_widgets=True)) + + def test_get_non_existint_dashbaord(self): + with app.test_client() as c, authenticated_user(c): + rv = c.get('/api/dashboards/not_existing') + self.assertEquals(rv.status_code, 404) + + def test_create_new_dashboard(self): + user_email = 'test@everything.me' + with app.test_client() as c, authenticated_user(c, user=user_email): + dashboard_name = 'Test Dashboard' + rv = json_request(c.post, '/api/dashboards', data={'name': dashboard_name}) + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.json['name'], 'Test Dashboard') + self.assertEquals(rv.json['user'], user_email) + self.assertEquals(rv.json['layout'], []) + + def test_update_dashboard(self): + d = DashboardFactory.create() + new_name = 'New Name' + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/dashboards/{0}'.format(d.id), + data={'name': new_name, 'layout': '[]'}) + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.json['name'], new_name) + + def test_delete_dashbaord(self): + d = DashboardFactory.create() + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.delete, '/api/dashboards/{0}'.format(d.slug)) + self.assertEquals(rv.status_code, 200) + + d = models.Dashboard.get_by_slug(d.slug) + self.assertTrue(d.is_archived) -class QueryAPITest(unittest.TestCase, AuthenticationTestMixin): +class QueryAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = ['/api/queries'] + super(QueryAPITest, self).setUp() -class QueryResultAPITest(unittest.TestCase, AuthenticationTestMixin): +class QueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = [] + super(QueryResultAPITest, self).setUp() -class JobAPITest(unittest.TestCase, AuthenticationTestMixin): +class JobAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = [] + super(JobAPITest, self).setUp() -class CsvQueryResultAPITest(unittest.TestCase, AuthenticationTestMixin): +class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): - self.paths = [] \ No newline at end of file + self.paths = [] + super(CsvQueryResultAPITest, self).setUp() \ No newline at end of file diff --git a/tests/models.py b/tests/models.py index c38839ed8..2f00df63a 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,31 +1,14 @@ -from unittest import TestCase -from redash import settings, db, app, models - -settings.DATABASE_CONFIG = { - 'name': 'rd_test', - 'engine': 'peewee.PostgresqlDatabase', - 'threadlocals': True -} -app.config['DATABASE'] = settings.DATABASE_CONFIG -db.load_database() +from tests import BaseTestCase, DashboardFactory -class DatabaseTestCase(TestCase): - def setUp(self): - models.create_db(True, True) - - def tearDown(self): - models.create_db(False, True) - - -class DashboardTest(DatabaseTestCase): +class DashboardTest(BaseTestCase): def test_appends_suffix_to_slug_when_duplicate(self): - d1 = models.Dashboard.create(name='test', user='arik', layout='') + d1 = DashboardFactory.create() self.assertEquals(d1.slug, 'test') - d2 = models.Dashboard.create(name='test', user='arik', layout='') + d2 = DashboardFactory.create() self.assertNotEquals(d1.slug, d2.slug) - d3 = models.Dashboard.create(name='test', user='arik', layout='') + d3 = DashboardFactory.create() self.assertNotEquals(d1.slug, d3.slug) self.assertNotEquals(d2.slug, d3.slug) \ No newline at end of file From d32799b2dcfd67807dfc5bcd363df42c48024ada Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 11:27:47 +0200 Subject: [PATCH 188/540] Add Visualization model. --- redash/models.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/redash/models.py b/redash/models.py index d1a9d8084..e571aa23a 100644 --- a/redash/models.py +++ b/redash/models.py @@ -56,7 +56,7 @@ class Query(db.Model): class Meta: db_table = 'queries' - def to_dict(self, with_result=True, with_stats=False): + def to_dict(self, with_result=True, with_stats=False, with_visualizations=False): d = { 'id': self.id, 'latest_query_data_id': self._data['latest_query_data'], @@ -77,6 +77,10 @@ class Query(db.Model): d['last_retrieved_at'] = self.last_retrieved_at d['times_retrieved'] = self.times_retrieved + if with_visualizations: + d['visualizations'] = [vis.to_dict(with_query=False) + for vis in self.visualizations] + if with_result and self._data['latest_query_data']: d['latest_query_data'] = self.latest_query_data.to_dict() @@ -162,9 +166,38 @@ class Dashboard(db.Model): return u"%s=%s" % (self.id, self.name) +class Visualization(db.Model): + id = peewee.PrimaryKeyField() + type = peewee.CharField(max_length=100) + query = peewee.ForeignKeyField(Query, related_name='visualizations') + name = peewee.CharField(max_length=255) + description = peewee.CharField(max_length=4096) + options = peewee.TextField() + + class Meta: + db_table = 'visualizations' + + def to_dict(self, with_query=True): + d = { + 'id': self.id, + 'type': self.type, + 'name': self.name, + 'description': self.description, + 'options': json.loads(self.options), + } + + if with_query: + d['query'] = self.query.to_dict() + + return d + + def __unicode__(self): + return u"%s=>%s" % (self.id, self.query_id) + + class Widget(db.Model): id = peewee.PrimaryKeyField() - query = peewee.ForeignKeyField(Query) + visualization = peewee.ForeignKeyField(Visualization, related_name='widgets') type = peewee.CharField(max_length=100) width = peewee.IntegerField() options = peewee.TextField() @@ -177,10 +210,10 @@ class Widget(db.Model): def to_dict(self): return { 'id': self.id, - 'query': self.query.to_dict(), 'type': self.type, 'width': self.width, 'options': json.loads(self.options), + 'visualization': self.visualization.to_dict(), 'dashboard_id': self._data['dashboard'] } @@ -189,7 +222,7 @@ class Widget(db.Model): def create_db(create_tables, drop_tables): - all_models = (QueryResult, Query, Dashboard, Widget) + all_models = (QueryResult, Query, Dashboard, Visualization, Widget) db.connect_db() From 2942d20ac38ff7bb369664f4ec44c929333fa9de Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 11:39:44 +0200 Subject: [PATCH 189/540] Visualization handlers. --- redash/controllers.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index b132120ae..2c8f0b5c7 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -104,7 +104,7 @@ class DashboardListAPI(BaseResource): class DashboardAPI(BaseResource): def get(self, dashboard_slug=None): - # TODO: prefetching of widgets and queries? + # TODO: prefetching of widgets, visualizations and query results? try: dashboard = models.Dashboard.get_by_slug(dashboard_slug) except models.Dashboard.DoesNotExist: @@ -182,6 +182,7 @@ class QueryListAPI(BaseResource): query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) query_def.pop('latest_query_data', None) + query_def.pop('visualizations', None) query_def['user'] = self.current_user query = models.Query(**query_def) @@ -212,7 +213,7 @@ class QueryAPI(BaseResource): def get(self, query_id): q = models.Query.get(models.Query.id == query_id) if q: - return q.to_dict() + return q.to_dict(with_visualizations=True) else: abort(404, message="Query not found.") @@ -220,6 +221,37 @@ api.add_resource(QueryListAPI, '/api/queries', endpoint='queries') api.add_resource(QueryAPI, '/api/queries/', endpoint='query') +class VisualizationListAPI(BaseResource): + def post(self): + kwargs = request.get_json(force=True) + kwargs['options'] = json.dumps(kwargs['options']) + + vis = data.models.Visualization(**kwargs) + vis.save() + + return vis.to_dict(with_query=False) + + +class VisualizationAPI(BaseResource): + def post(self, visualization_id): + kwargs = request.get_json(force=True) + kwargs['options'] = json.dumps(kwargs['options']) + + vis = models.Visualization(**kwargs) + fields = kwargs.keys() + fields.remove('id') + vis.save(only=fields) + + return vis.to_dict(with_query=False) + + def delete(self, visualization_id): + vis = models.Visualization.get(models.Visualization.id == visualization_id) + vis.delete() + +api.add_resource(VisualizationListAPI, '/api/visualizations', endpoint='visualizations') +api.add_resource(VisualizationAPI, '/api/visualizations/', endpoint='visualization') + + class QueryResultListAPI(BaseResource): def post(self): params = request.json From 250aa17e631e73326aede369197979a113860f43 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 12:10:13 +0200 Subject: [PATCH 190/540] Fix: bring back support for MySQL --- manage.py | 2 +- redash/data/manager.py | 8 ++++---- {rd_service => redash}/data/query_runner_mysql.py | 3 +-- redash/settings_example.py | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) rename {rd_service => redash}/data/query_runner_mysql.py (97%) diff --git a/manage.py b/manage.py index a379b0a44..19605bcc1 100755 --- a/manage.py +++ b/manage.py @@ -20,7 +20,7 @@ def start_workers(): logging.info("Cleaning old workers: %s", old_workers) - data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_STRING) + data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: diff --git a/redash/data/manager.py b/redash/data/manager.py index 007e0a88e..46e12f9ab 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -149,15 +149,15 @@ class Manager(object): return data - def start_workers(self, workers_count, connection_string): + def start_workers(self, workers_count, connection_type, connection_string): if self.workers: return self.workers - + if getattr(settings, 'CONNECTION_ADAPTER', None) == "mysql": - import query_runner_mysql + from redash.data import query_runner_mysql runner = query_runner_mysql.mysql(connection_string) else: - import query_runner + from redash.data import query_runner runner = query_runner.redshift(connection_string) redis_connection_params = self.redis_connection.connection_pool.connection_kwargs diff --git a/rd_service/data/query_runner_mysql.py b/redash/data/query_runner_mysql.py similarity index 97% rename from rd_service/data/query_runner_mysql.py rename to redash/data/query_runner_mysql.py index ff849598c..de9b7d7bf 100644 --- a/rd_service/data/query_runner_mysql.py +++ b/redash/data/query_runner_mysql.py @@ -10,8 +10,7 @@ import logging import json import MySQLdb import sys -import select -from .utils import JSONEncoder +from redash.utils import JSONEncoder def mysql(connection_string): if connection_string.endswith(';'): diff --git a/redash/settings_example.py b/redash/settings_example.py index 43056090f..1f49cdb99 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -4,7 +4,7 @@ Example settings module. You should make your own copy as settings.py and enter REDIS_URL = "redis://localhost:6379" # Either "pg" or "mysql" -CONNECTION_ADAPTER = "mysql" +CONNECTION_ADAPTER = "pg" # Connection string for the database that is used to run queries against # -- example mysql CONNECTION_STRING = "Server=;User=;Pwd=;Database=" # -- example pg CONNECTION_STRING = "user= password= host= port=5439 dbname=" From 6765d7d89fc91dbe2f21d9dec1e978edd6c34557 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 5 Feb 2014 12:20:16 +0200 Subject: [PATCH 191/540] Migrations folder --- migrations/add_created_at_field.py | 13 +++++++++++++ .../create_visualizations.py | 3 +-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 migrations/add_created_at_field.py rename rd_service/migrate.py => migrations/create_visualizations.py (98%) diff --git a/migrations/add_created_at_field.py b/migrations/add_created_at_field.py new file mode 100644 index 000000000..a3ff7d665 --- /dev/null +++ b/migrations/add_created_at_field.py @@ -0,0 +1,13 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.add_column(models.Dashboard, models.Dashboard.created_at, 'created_at') + migrator.add_column(models.Widget, models.Widget.created_at, 'created_at') + + db.close_db(None) \ No newline at end of file diff --git a/rd_service/migrate.py b/migrations/create_visualizations.py similarity index 98% rename from rd_service/migrate.py rename to migrations/create_visualizations.py index 2930b7b4a..acb2e86ed 100644 --- a/rd_service/migrate.py +++ b/migrations/create_visualizations.py @@ -1,6 +1,5 @@ import json -import settings -from data.models import * +from redash import models # first run: From 3471b9853e670929c3b2456c8ca76e6a9c2072f0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 11:09:39 +0200 Subject: [PATCH 192/540] Update circle config to run python tests & cache packages. --- circle.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 9b4ecfb6b..23c414e72 100644 --- a/circle.yml +++ b/circle.yml @@ -8,8 +8,14 @@ machine: dependencies: pre: - make deps - - pip install requests + - pip install requests coverage nose + - pip install -r requirements.txt + cache_directories: + - rd_ui/node_modules/ + - rd_ui/app/bower_components/ test: + override: + - make test post: - make pack deployment: From c0ca6020179e380101e8fe636b61b0e28ee76b07 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 11:10:04 +0200 Subject: [PATCH 193/540] Factories for all models for tests. --- tests/__init__.py | 37 +++------------------------------- tests/factories.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++ tests/models.py | 9 +++++---- 3 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 tests/factories.py diff --git a/tests/__init__.py b/tests/__init__.py index 6618283a3..04b32415d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -11,44 +11,13 @@ settings.DATABASE_CONFIG = { app.config['DATABASE'] = settings.DATABASE_CONFIG db.load_database() - -def model_factory(model, **kwargs): - def factory(**properties): - kwargs.update(properties) - - return model(**kwargs) - - return factory - - -class ModelFactory(object): - def __init__(self, model, **kwargs): - self.model = model - self.kwargs = kwargs - - def _get_kwargs(self, override_kwargs): - kwargs = self.kwargs.copy() - kwargs.update(override_kwargs) - return kwargs - - def instance(self, **override_kwargs): - kwargs = self._get_kwargs(override_kwargs) - - return self.model(**kwargs) - - def create(self, **override_kwargs): - kwargs = self._get_kwargs(override_kwargs) - - return self.model.create(**kwargs) - -DashboardFactory = ModelFactory(redash.models.Dashboard, - name='test', user='test@everything.me', layout='[]') - -dashboard_factory = model_factory(redash.models.Dashboard, name='test', user='test@everything.me', layout='[]') +for model in redash.models.all_models: + model._meta.database = db.database class BaseTestCase(TestCase): def setUp(self): + redash.models.create_db(True, True) def tearDown(self): diff --git a/tests/factories.py b/tests/factories.py new file mode 100644 index 000000000..5e96197a0 --- /dev/null +++ b/tests/factories.py @@ -0,0 +1,50 @@ +import redash.models + + +class ModelFactory(object): + def __init__(self, model, **kwargs): + self.model = model + self.kwargs = kwargs + + def _get_kwargs(self, override_kwargs): + kwargs = self.kwargs.copy() + kwargs.update(override_kwargs) + + for key, arg in kwargs.items(): + if callable(arg): + kwargs[key] = arg() + + return kwargs + + def instance(self, **override_kwargs): + kwargs = self._get_kwargs(override_kwargs) + + return self.model(**kwargs) + + def create(self, **override_kwargs): + kwargs = self._get_kwargs(override_kwargs) + return self.model.create(**kwargs) + +dashboard_factory = ModelFactory(redash.models.Dashboard, + name='test', user='test@everything.me', layout='[]') + +query_factory = ModelFactory(redash.models.Query, + name='New Query', + description='', + query='SELECT 1', + ttl=-1, + user='test@everything.me') + +visualization_factory = ModelFactory(redash.models.Visualization, + type='CHART', + query=query_factory.create, + name='Chart', + description='', + options='{}') + +widget_factory = ModelFactory(redash.models.Widget, + type='chart', + width=1, + options='{}', + dashboard=dashboard_factory.create, + visualization=visualization_factory.create) diff --git a/tests/models.py b/tests/models.py index 2f00df63a..a408b51ed 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,14 +1,15 @@ -from tests import BaseTestCase, DashboardFactory +from tests import BaseTestCase +from factories import dashboard_factory class DashboardTest(BaseTestCase): def test_appends_suffix_to_slug_when_duplicate(self): - d1 = DashboardFactory.create() + d1 = dashboard_factory.create() self.assertEquals(d1.slug, 'test') - d2 = DashboardFactory.create() + d2 = dashboard_factory.create() self.assertNotEquals(d1.slug, d2.slug) - d3 = DashboardFactory.create() + d3 = dashboard_factory.create() self.assertNotEquals(d1.slug, d3.slug) self.assertNotEquals(d2.slug, d3.slug) \ No newline at end of file From 199cddfbdbc107ad549f0406b35df872998733cd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 11:10:41 +0200 Subject: [PATCH 194/540] Tests for Query, Widget and Dashboard controllers. --- redash/controllers.py | 28 +++++----- redash/models.py | 19 ++++--- redash/utils.py | 4 ++ tests/controllers.py | 118 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 141 insertions(+), 28 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 2c8f0b5c7..8c3b26f4f 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -135,6 +135,9 @@ class WidgetListAPI(BaseResource): def post(self): widget_properties = request.get_json(force=True) widget_properties['options'] = json.dumps(widget_properties['options']) + widget_properties.pop('id', None) + widget_properties['dashboard'] = widget_properties.pop('dashboard_id') + widget_properties['visualization'] = widget_properties.pop('visualization_id') widget = models.Widget(**widget_properties) widget.save() @@ -169,7 +172,7 @@ class WidgetAPI(BaseResource): widget.dashboard.layout = json.dumps(layout) widget.dashboard.save() - widget.delete() + widget.delete_instance() api.add_resource(WidgetListAPI, '/api/widgets', endpoint='widgets') api.add_resource(WidgetAPI, '/api/widgets/', endpoint='widget') @@ -177,10 +180,7 @@ api.add_resource(WidgetAPI, '/api/widgets/', endpoint='widget') class QueryListAPI(BaseResource): def post(self): - query_def = request.json - if 'created_at' in query_def: - query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) - + query_def = request.get_json(force=True) query_def.pop('latest_query_data', None) query_def.pop('visualizations', None) @@ -196,17 +196,15 @@ class QueryListAPI(BaseResource): class QueryAPI(BaseResource): def post(self, query_id): - query_def = request.json - if 'created_at' in query_def: - query_def['created_at'] = dateutil.parser.parse(query_def['created_at']) - + query_def = request.get_json(force=True) + query_def.pop('created_at', None) query_def.pop('latest_query_data', None) + query_def.pop('id', None) - query = models.Query(**query_def) - fields = query_def.keys() - fields.remove('id') - # model.save(only=model.dirty_fields) - query.save(only=fields) + update = models.Query.update(**query_def).where(models.Query.id == query_id) + update.execute() + + query = models.Query.get_by_id(query_id) return query.to_dict(with_result=False) @@ -246,7 +244,7 @@ class VisualizationAPI(BaseResource): def delete(self, visualization_id): vis = models.Visualization.get(models.Visualization.id == visualization_id) - vis.delete() + vis.delete_instance() api.add_resource(VisualizationListAPI, '/api/visualizations', endpoint='visualizations') api.add_resource(VisualizationAPI, '/api/visualizations/', endpoint='visualization') diff --git a/redash/models.py b/redash/models.py index e571aa23a..9a8fe0b89 100644 --- a/redash/models.py +++ b/redash/models.py @@ -16,6 +16,12 @@ from redash import db, utils # return '' % (self.name, self.email) +class BaseModel(db.Model): + @classmethod + def get_by_id(cls, model_id): + return cls.get(cls.id == model_id) + + class QueryResult(db.Model): id = peewee.PrimaryKeyField() query_hash = peewee.CharField(max_length=32, index=True) @@ -41,7 +47,7 @@ class QueryResult(db.Model): return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) -class Query(db.Model): +class Query(BaseModel): id = peewee.PrimaryKeyField() latest_query_data = peewee.ForeignKeyField(QueryResult, null=True) name = peewee.CharField(max_length=255) @@ -59,7 +65,7 @@ class Query(db.Model): def to_dict(self, with_result=True, with_stats=False, with_visualizations=False): d = { 'id': self.id, - 'latest_query_data_id': self._data['latest_query_data'], + 'latest_query_data_id': self._data.get('latest_query_data', None), 'name': self.name, 'description': self.description, 'query': self.query, @@ -81,7 +87,7 @@ class Query(db.Model): d['visualizations'] = [vis.to_dict(with_query=False) for vis in self.visualizations] - if with_result and self._data['latest_query_data']: + if with_result and self.latest_query_data: d['latest_query_data'] = self.latest_query_data.to_dict() return d @@ -198,7 +204,7 @@ class Visualization(db.Model): class Widget(db.Model): id = peewee.PrimaryKeyField() visualization = peewee.ForeignKeyField(Visualization, related_name='widgets') - type = peewee.CharField(max_length=100) + #type = peewee.CharField(max_length=100) width = peewee.IntegerField() options = peewee.TextField() dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) @@ -210,7 +216,6 @@ class Widget(db.Model): def to_dict(self): return { 'id': self.id, - 'type': self.type, 'width': self.width, 'options': json.loads(self.options), 'visualization': self.visualization.to_dict(), @@ -220,10 +225,10 @@ class Widget(db.Model): def __unicode__(self): return u"%s=>%s" % (self.id, self.dashboard_id) +all_models = (QueryResult, Query, Dashboard, Visualization, Widget) + def create_db(create_tables, drop_tables): - all_models = (QueryResult, Query, Dashboard, Visualization, Widget) - db.connect_db() for model in all_models: diff --git a/redash/utils.py b/redash/utils.py index 0d28a26c7..0e6c2b24f 100644 --- a/redash/utils.py +++ b/redash/utils.py @@ -36,6 +36,10 @@ class JSONEncoder(json.JSONEncoder): super(JSONEncoder, self).default(o) +def json_dumps(data): + return json.dumps(data, cls=JSONEncoder) + + class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", diff --git a/tests/controllers.py b/tests/controllers.py index f3c5d2686..8c4bae450 100644 --- a/tests/controllers.py +++ b/tests/controllers.py @@ -1,7 +1,9 @@ from contextlib import contextmanager import json -from tests import BaseTestCase, DashboardFactory +from tests import BaseTestCase +from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory from redash import app, models +from redash.utils import json_dumps @contextmanager @@ -14,7 +16,7 @@ def authenticated_user(c, user='test@example.com', name='John Test'): def json_request(method, path, data=None): if data: - response = method(path, data=json.dumps(data)) + response = method(path, data=json_dumps(data)) else: response = method(path) @@ -66,7 +68,7 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): super(DashboardAPITest, self).setUp() def test_get_dashboard(self): - d1 = DashboardFactory.create() + d1 = dashboard_factory.create() with app.test_client() as c, authenticated_user(c): rv = c.get('/api/dashboards/{0}'.format(d1.slug)) self.assertEquals(rv.status_code, 200) @@ -88,7 +90,7 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.json['layout'], []) def test_update_dashboard(self): - d = DashboardFactory.create() + d = dashboard_factory.create() new_name = 'New Name' with app.test_client() as c, authenticated_user(c): rv = json_request(c.post, '/api/dashboards/{0}'.format(d.id), @@ -96,8 +98,8 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.status_code, 200) self.assertEquals(rv.json['name'], new_name) - def test_delete_dashbaord(self): - d = DashboardFactory.create() + def test_delete_dashboard(self): + d = dashboard_factory.create() with app.test_client() as c, authenticated_user(c): rv = json_request(c.delete, '/api/dashboards/{0}'.format(d.slug)) self.assertEquals(rv.status_code, 200) @@ -106,11 +108,115 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): self.assertTrue(d.is_archived) +class WidgetAPITest(BaseTestCase): + def create_widget(self, dashboard, visualization, width=1): + data = { + 'visualization_id': visualization.id, + 'dashboard_id': dashboard.id, + 'options': {}, + 'width': width + } + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/widgets', data=data) + + return rv + + def test_create_widget(self): + dashboard = dashboard_factory.create() + vis = visualization_factory.create() + + rv = self.create_widget(dashboard, vis) + self.assertEquals(rv.status_code, 200) + + dashboard = models.Dashboard.get(models.Dashboard.id == dashboard.id) + self.assertEquals(unicode(rv.json['layout']), dashboard.layout) + + self.assertEquals(dashboard.widgets, 1) + self.assertEquals(rv.json['layout'], [[rv.json['widget']['id']]]) + self.assertEquals(rv.json['new_row'], True) + + rv2 = self.create_widget(dashboard, vis) + self.assertEquals(dashboard.widgets, 2) + self.assertEquals(rv2.json['layout'], + [[rv.json['widget']['id'], rv2.json['widget']['id']]]) + self.assertEquals(rv2.json['new_row'], False) + + rv3 = self.create_widget(dashboard, vis) + self.assertEquals(rv3.json['new_row'], True) + rv4 = self.create_widget(dashboard, vis, width=2) + self.assertEquals(rv4.json['layout'], + [[rv.json['widget']['id'], rv2.json['widget']['id']], + [rv3.json['widget']['id']], + [rv4.json['widget']['id']]]) + self.assertEquals(rv4.json['new_row'], True) + + def test_delete_widget(self): + widget = widget_factory.create() + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.delete, '/api/widgets/{0}'.format(widget.id)) + + self.assertEquals(rv.status_code, 200) + dashboard = models.Dashboard.get_by_slug(widget.dashboard.slug) + self.assertEquals(dashboard.widgets.count(), 0) + self.assertEquals(dashboard.layout, '[]') + + # TODO: test how it updates the layout + + class QueryAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): self.paths = ['/api/queries'] super(QueryAPITest, self).setUp() + def test_update_query(self): + query = query_factory.create() + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/queries/{0}'.format(query.id), data={'name': 'Testing'}) + self.assertEqual(rv.status_code, 200) + self.assertEquals(rv.json['name'], 'Testing') + + def test_create_query(self): + user = 'test@everything.me' + query_data = { + 'name': 'Testing', + 'description': 'Description', + 'query': 'SELECT 1', + 'ttl': 3600 + } + + with app.test_client() as c, authenticated_user(c, user=user): + rv = json_request(c.post, '/api/queries', data=query_data) + + self.assertEquals(rv.status_code, 200) + self.assertDictContainsSubset(query_data, rv.json) + self.assertEquals(rv.json['user'], user) + self.assertIsNotNone(rv.json['api_key']) + self.assertIsNotNone(rv.json['query_hash']) + + def test_get_query(self): + query = query_factory.create() + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.get, '/api/queries/{0}'.format(query.id)) + + self.assertEquals(rv.status_code, 200) + d = query.to_dict(with_visualizations=True) + d.pop('created_at') + self.assertDictContainsSubset(d, rv.json) + + def test_get_all_queries(self): + queries = [query_factory.create() for _ in range(10)] + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.get, '/api/queries') + + self.assertEquals(rv.status_code, 200) + self.assertEquals(len(rv.json), 10) + + class QueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): From 78e748548c1074f5239730a8d2d7146fda2e3329 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 20:14:11 +0200 Subject: [PATCH 195/540] Update circle config to create settings.py file. --- circle.yml | 2 ++ redash/settings_example.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 23c414e72..0f0aa0a53 100644 --- a/circle.yml +++ b/circle.yml @@ -10,6 +10,7 @@ dependencies: - make deps - pip install requests coverage nose - pip install -r requirements.txt + - cp redash/settings_example.py redash/settings.py cache_directories: - rd_ui/node_modules/ - rd_ui/app/bower_components/ @@ -17,6 +18,7 @@ test: override: - make test post: + - rm redash/settings.py - make pack deployment: github: diff --git a/redash/settings_example.py b/redash/settings_example.py index 1f49cdb99..92c1488fe 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -11,7 +11,7 @@ CONNECTION_ADAPTER = "pg" CONNECTION_STRING = "user= password= host= port=5439 dbname=" # Connection settings for re:dash's own database (where we store the queries, results, etc) DATABASE_CONFIG = { - 'name': 'postgres', + 'name': 'circle_test', 'engine': 'peewee.PostgresqlDatabase', } # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed From b4f88196dc2b1811d7058eac48d9577aa9a31e8a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 20:19:01 +0200 Subject: [PATCH 196/540] Use same database name in tests as CircleCI (until we add config for tests). --- redash/settings_example.py | 2 +- tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/settings_example.py b/redash/settings_example.py index 92c1488fe..1f49cdb99 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -11,7 +11,7 @@ CONNECTION_ADAPTER = "pg" CONNECTION_STRING = "user= password= host= port=5439 dbname=" # Connection settings for re:dash's own database (where we store the queries, results, etc) DATABASE_CONFIG = { - 'name': 'circle_test', + 'name': 'postgres', 'engine': 'peewee.PostgresqlDatabase', } # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed diff --git a/tests/__init__.py b/tests/__init__.py index 04b32415d..452449f60 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,7 +4,7 @@ import redash.models # TODO: this isn't pretty... :-) settings.DATABASE_CONFIG = { - 'name': 'rd_test', + 'name': 'circle_test', 'engine': 'peewee.PostgresqlDatabase', 'threadlocals': True } From fb9f814b00989048b4fb24b57b1bdcc2281d03e7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 20:49:05 +0200 Subject: [PATCH 197/540] Visualization API tests. --- redash/controllers.py | 17 ++++++++++------- redash/models.py | 2 +- tests/controllers.py | 36 ++++++++++++++++++++++++++++++++++++ tests/factories.py | 8 ++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 8c3b26f4f..faf268523 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -223,8 +223,9 @@ class VisualizationListAPI(BaseResource): def post(self): kwargs = request.get_json(force=True) kwargs['options'] = json.dumps(kwargs['options']) + kwargs['query'] = kwargs.pop('query_id') - vis = data.models.Visualization(**kwargs) + vis = models.Visualization(**kwargs) vis.save() return vis.to_dict(with_query=False) @@ -233,12 +234,14 @@ class VisualizationListAPI(BaseResource): class VisualizationAPI(BaseResource): def post(self, visualization_id): kwargs = request.get_json(force=True) - kwargs['options'] = json.dumps(kwargs['options']) + if 'options' in kwargs: + kwargs['options'] = json.dumps(kwargs['options']) + kwargs.pop('id', None) - vis = models.Visualization(**kwargs) - fields = kwargs.keys() - fields.remove('id') - vis.save(only=fields) + update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id) + update.execute() + + vis = models.Visualization.get_by_id(visualization_id) return vis.to_dict(with_query=False) @@ -280,7 +283,7 @@ class CsvQueryResultsAPI(BaseResource): if not query_result_id: query = models.Query.get(models.Query.id == query_id) if query: - query_result_id = query.latest_query_data_id + query_result_id = query._data['latest_query_data'] query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id) if query_result: diff --git a/redash/models.py b/redash/models.py index 9a8fe0b89..c265920f1 100644 --- a/redash/models.py +++ b/redash/models.py @@ -172,7 +172,7 @@ class Dashboard(db.Model): return u"%s=%s" % (self.id, self.name) -class Visualization(db.Model): +class Visualization(BaseModel): id = peewee.PrimaryKeyField() type = peewee.CharField(max_length=100) query = peewee.ForeignKeyField(Query, related_name='visualizations') diff --git a/tests/controllers.py b/tests/controllers.py index 8c4bae450..287cf0f3a 100644 --- a/tests/controllers.py +++ b/tests/controllers.py @@ -217,6 +217,42 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(len(rv.json), 10) +class VisualizationAPITest(BaseTestCase): + def test_create_visualization(self): + query = query_factory.create() + data = { + 'query_id': query.id, + 'name': 'Chart', + 'description':'', + 'options': {}, + 'type': 'CHART' + } + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/visualizations', data=data) + + self.assertEquals(rv.status_code, 200) + data.pop('query_id') + self.assertDictContainsSubset(data, rv.json) + + def test_delete_visualization(self): + visualization = visualization_factory.create() + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.delete, '/api/visualizations/{0}'.format(visualization.id)) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(models.Visualization.select().count(), 0) + + def test_update_visualization(self): + visualization = visualization_factory.create() + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/visualizations/{0}'.format(visualization.id), + data={'name': 'After Update'}) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.json['name'], 'After Update') + class QueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): diff --git a/tests/factories.py b/tests/factories.py index 5e96197a0..3f48ff428 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -1,3 +1,4 @@ +import datetime import redash.models @@ -35,6 +36,13 @@ query_factory = ModelFactory(redash.models.Query, ttl=-1, user='test@everything.me') +query_result_factory = ModelFactory(redash.models.QueryResult, + data='{}', + runtime=1, + retrieved_at=datetime.datetime.now(), + query=query_factory.create, + query_hash='') + visualization_factory = ModelFactory(redash.models.Visualization, type='CHART', query=query_factory.create, From 0f8bbdc9f2e909c6f5759bffe21a976380db57d2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 21:12:02 +0200 Subject: [PATCH 198/540] Create default visualization. --- redash/controllers.py | 2 ++ redash/models.py | 6 ++++++ tests/controllers.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/redash/controllers.py b/redash/controllers.py index faf268523..976f5f9ff 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -188,6 +188,8 @@ class QueryListAPI(BaseResource): query = models.Query(**query_def) query.save() + query.create_default_visualizations() + return query.to_dict(with_result=False) def get(self): diff --git a/redash/models.py b/redash/models.py index c265920f1..6e4c2214d 100644 --- a/redash/models.py +++ b/redash/models.py @@ -62,6 +62,12 @@ class Query(BaseModel): class Meta: db_table = 'queries' + def create_default_visualizations(self): + table_visualization = Visualization(query=self, name="Table", + description=self.description, + type="TABLE", options="{}") + table_visualization.save() + def to_dict(self, with_result=True, with_stats=False, with_visualizations=False): d = { 'id': self.id, diff --git a/tests/controllers.py b/tests/controllers.py index 287cf0f3a..1b3032315 100644 --- a/tests/controllers.py +++ b/tests/controllers.py @@ -196,6 +196,9 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): self.assertIsNotNone(rv.json['api_key']) self.assertIsNotNone(rv.json['query_hash']) + query = models.Query.get_by_id(rv.json['id']) + self.assertEquals(len(list(query.visualizations)), 1) + def test_get_query(self): query = query_factory.create() From d4ca903a07a4ea44a1d4b6fc08568fca50f2a6ce Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 21:21:52 +0200 Subject: [PATCH 199/540] CirlceCI badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ee5b67cf8..c51744f63 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +![Build Status](https://circleci.com/gh/everythingme/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") # [_re:dash_](https://github.com/everythingme/redash) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. From 27bf2e642b00c40f4d8437d6ec6ad1d3606928a8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 21:25:18 +0200 Subject: [PATCH 200/540] Coveralls badge --- README.md | 2 +- circle.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c51744f63..3b725b00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Build Status](https://circleci.com/gh/everythingme/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") +![Build Status](https://circleci.com/gh/everythingme/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") [![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) # [_re:dash_](https://github.com/everythingme/redash) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. diff --git a/circle.yml b/circle.yml index 0f0aa0a53..ce37e51af 100644 --- a/circle.yml +++ b/circle.yml @@ -8,7 +8,7 @@ machine: dependencies: pre: - make deps - - pip install requests coverage nose + - pip install requests coverage nose coveralls - pip install -r requirements.txt - cp redash/settings_example.py redash/settings.py cache_directories: @@ -18,6 +18,7 @@ test: override: - make test post: + - coveralls - rm redash/settings.py - make pack deployment: From f571e8ac6e7a7d658954855aaa82cbd1a99eeece Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 21:26:51 +0200 Subject: [PATCH 201/540] Fix build status image link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b725b00f..df46476a7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Build Status](https://circleci.com/gh/everythingme/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") [![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) +![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") [![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) # [_re:dash_](https://github.com/everythingme/redash) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. From 1bce924d83c62af53c7578f68d88ada67bc561a3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Feb 2014 21:27:43 +0200 Subject: [PATCH 202/540] Readme formatting --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index df46476a7..e5a2499bb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") [![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) # [_re:dash_](https://github.com/everythingme/redash) +![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") +[![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. From a6ab0ff2aa044dff350989b578838f1c44374bfe Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 8 Feb 2014 21:01:21 +0200 Subject: [PATCH 203/540] Fix unicode representatino of Widget and Visualization models. --- redash/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/models.py b/redash/models.py index 6e4c2214d..f29f59df8 100644 --- a/redash/models.py +++ b/redash/models.py @@ -204,7 +204,7 @@ class Visualization(BaseModel): return d def __unicode__(self): - return u"%s=>%s" % (self.id, self.query_id) + return u"%s %s" % (self.id, self.type) class Widget(db.Model): @@ -229,7 +229,7 @@ class Widget(db.Model): } def __unicode__(self): - return u"%s=>%s" % (self.id, self.dashboard_id) + return u"%s" % self.id all_models = (QueryResult, Query, Dashboard, Visualization, Widget) From d084b5a03c637d5d93d76d76b7c4de08e328f456 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 8 Feb 2014 21:01:48 +0200 Subject: [PATCH 204/540] Bring back type to Widget definition. --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index f29f59df8..57e50f997 100644 --- a/redash/models.py +++ b/redash/models.py @@ -210,7 +210,7 @@ class Visualization(BaseModel): class Widget(db.Model): id = peewee.PrimaryKeyField() visualization = peewee.ForeignKeyField(Visualization, related_name='widgets') - #type = peewee.CharField(max_length=100) + type = peewee.CharField(max_length=100, null=True) # unused; kept for backward compatability width = peewee.IntegerField() options = peewee.TextField() dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) From f9b6aca8e8f6c25c52c59eaae84a8ed749ef27b6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 8 Feb 2014 21:16:36 +0200 Subject: [PATCH 205/540] Prefetching for widgets/visualizations/queries/query resutls when getting dashboard. --- redash/controllers.py | 1 - redash/models.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 976f5f9ff..655df8d8e 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -104,7 +104,6 @@ class DashboardListAPI(BaseResource): class DashboardAPI(BaseResource): def get(self, dashboard_slug=None): - # TODO: prefetching of widgets, visualizations and query results? try: dashboard = models.Dashboard.get_by_slug(dashboard_slug) except models.Dashboard.DoesNotExist: diff --git a/redash/models.py b/redash/models.py index 57e50f997..1d4fe39c0 100644 --- a/redash/models.py +++ b/redash/models.py @@ -145,7 +145,9 @@ class Dashboard(db.Model): layout = json.loads(self.layout) if with_widgets: - widgets = {w.id: w.to_dict() for w in self.widgets} + widgets = Widget.select(Widget, Visualization, Query, QueryResult).\ + where(Widget.dashboard == self.id).join(Visualization).join(Query).join(QueryResult) + widgets = {w.id: w.to_dict() for w in widgets} widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) else: widgets_layout = None From 9055865e1cca6260f470a22451a17100f35689e0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 14:33:05 +0200 Subject: [PATCH 206/540] Migratino to set Widget.type and Widget.query_id to nullables --- migrations/change_query_id_on_widgets_to_null.py | 13 +++++++++++++ redash/models.py | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 migrations/change_query_id_on_widgets_to_null.py diff --git a/migrations/change_query_id_on_widgets_to_null.py b/migrations/change_query_id_on_widgets_to_null.py new file mode 100644 index 000000000..580bed29f --- /dev/null +++ b/migrations/change_query_id_on_widgets_to_null.py @@ -0,0 +1,13 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.set_nullable(models.Widget, models.Widget.query_id, True) + migrator.set_nullable(models.Widget, models.Widget.type, True) + + db.close_db(None) \ No newline at end of file diff --git a/redash/models.py b/redash/models.py index 1d4fe39c0..d9378d63d 100644 --- a/redash/models.py +++ b/redash/models.py @@ -212,12 +212,16 @@ class Visualization(BaseModel): class Widget(db.Model): id = peewee.PrimaryKeyField() visualization = peewee.ForeignKeyField(Visualization, related_name='widgets') - type = peewee.CharField(max_length=100, null=True) # unused; kept for backward compatability + width = peewee.IntegerField() options = peewee.TextField() dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) created_at = peewee.DateTimeField(default=datetime.datetime.now) + # unused; kept for backward compatability: + type = peewee.CharField(max_length=100, null=True) + query_id = peewee.IntegerField(null=True) + class Meta: db_table = 'widgets' From 970e0e2d044d821558812b9c4f58cc45ba1793fc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 14:33:52 +0200 Subject: [PATCH 207/540] Fix: format_sql api call wasn't working. --- redash/controllers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 655df8d8e..c89c346ad 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -10,7 +10,6 @@ import json import numbers import cStringIO import datetime -import dateutil.parser from flask import g, render_template, send_from_directory, make_response, request, jsonify from flask.ext.restful import Resource, abort @@ -71,8 +70,8 @@ def status_api(): @app.route('/api/queries/format', methods=['POST']) @auth.required -def format_sql_query(self): - arguments = json.loads(self.request.body) +def format_sql_query(): + arguments = request.get_json(force=True) query = arguments.get("query", "") return sqlparse.format(query, reindent=True, keyword_case='upper') From f265d9174a8388e6916b51254a2eb1488914dea2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 14:34:27 +0200 Subject: [PATCH 208/540] Fix: POST api/queries fields cleanup logic --- redash/controllers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index c89c346ad..15d77c950 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -179,8 +179,9 @@ api.add_resource(WidgetAPI, '/api/widgets/', endpoint='widget') class QueryListAPI(BaseResource): def post(self): query_def = request.get_json(force=True) - query_def.pop('latest_query_data', None) - query_def.pop('visualizations', None) + # id, created_at, api_key + for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: + query_def.pop(field, None) query_def['user'] = self.current_user query = models.Query(**query_def) @@ -197,9 +198,11 @@ class QueryListAPI(BaseResource): class QueryAPI(BaseResource): def post(self, query_id): query_def = request.get_json(force=True) - query_def.pop('created_at', None) - query_def.pop('latest_query_data', None) - query_def.pop('id', None) + for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: + query_def.pop(field, None) + + if 'latest_query_data_id' in query_def: + query_def['latest_query_data'] = query_def.pop('latest_query_data_id') update = models.Query.update(**query_def).where(models.Query.id == query_id) update.execute() From bcf854604bcc8523d850b473d85094801fa7b40b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 14:37:48 +0200 Subject: [PATCH 209/540] Fix: bring back TABLE renderer to VisualizationRenderer. --- rd_ui/app/scripts/query_fiddle/renderers.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 786442cb6..e747ac182 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -8,6 +8,7 @@ renderers.directive('visualizationRenderer', function() { queryResult: '=' }, template: '
        ' + + '' + '' + '' + '
        ', From 1b1416153542e00faaf374c25e5dd1f5932ced96 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 14:48:15 +0200 Subject: [PATCH 210/540] Show query name in dashboard editor --- rd_ui/app/scripts/directives.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index beb9e675d..a1b85e91b 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -177,7 +177,7 @@ row: rowIndex+1, ySize: 1, xSize: widget.width, - name: widget.visualization.name + name: widget.visualization.query.name }); }); }); From d954eb63efb6e6a2f6ce283856a96e954eb8ec3f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 15:00:42 +0200 Subject: [PATCH 211/540] Update getting started instructions in the README. --- README.md | 57 ++++++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index e5a2499bb..93445f8c0 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Due to Heroku dev plan limits, it has a small database of flights (see schema [h ## Technology +* Python * [AngularJS](http://angularjs.org/) -* [Tornado](http://tornadoweb.org) * [PostgreSQL](http://www.postgresql.org/) / [AWS Redshift](http://aws.amazon.com/redshift/) * [Redis](http://redis.io) @@ -47,62 +47,33 @@ It's very likely that in the future we will switch to [D3.js](http://d3js.org/) ## Getting Started -1. Clone the repo: -```bash -git clone git@github.com:EverythingMe/redash.git -``` -2. Create settings file from the example one (& update relevant settings): -```bash -cp rd_service/settings_example.py rd_service/settings.py -``` -It's highly recommended that the user you use to connect to the data database (the one you query) is read-only. -3. Create the operational databases from rd_service/data/tables.sql -3. Install `npm` packages (mainly: Bower & Grunt): -```bash -cd rd_ui -npm install -``` -4. Install `bower` packages: -```bash -bower install -``` -5. Build the UI: -```bash -grunt build -``` -6. Install PIP packages: -```bash -pip install -r ../rd_service/requirements.txt -``` -6. Start the API server: -```bash -cd ../rd_service -python server.py -``` -7. Start the workers: -```bash -python cli.py worker -``` +1. Download the [latest release](https://github.com/everythingme/redash/releases). +2. Make sure you have `Python` v2.7, `pip`, PostgreSQL and Redis installed. +3. Install Python requirements: `pip install -r requirements.txt`. +4. Make a copy of the examples settings file: `cp redash/settings_example.py redash/settings.py` and edit the relevant settings. +5. Create database: `./manage.py db --create-tables`. +6. Start the web server: `./manage.py server`. +7. Start the worker: `./manage.py worker`. 8. Open `http://localhost:8888/` and query away. -## Roadmap +**Need help setting re:dash or one of the dependencies up?** Ping @arikfr on the IRC #redash channel or send a message to the [mailing list](https://groups.google.com/forum/#!forum/redash-users), and he will gladly help. -We plan to release new minor version every 2-3 weeks. Of course, if we get additional help from contributors it will help speed things up. +## Roadmap Below you can see the "big" features of the next 3 releases (for full list, click on the link): ### [v0.2](https://github.com/EverythingMe/redash/issues?milestone=1&state=open) - Ability to generate multiple visualizations for a single query (dataset) in a more flexible way than today. Also easier extensbility points to add additional visualizations. -- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors). -- UI Improvements (better notifications & flows, improved queries page) -- Comments on queries. +- Support for API access using API keys, instead of Google Login. +- UI Improvements (better notifications & flows, improved queries page) ### [v0.3](https://github.com/EverythingMe/redash/issues?milestone=2&state=open) -- Support for API access using API keys, instead of Google Login. +- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors). - Multiple databases support (including other database type than PostgreSQL). - Scheduled reports by email. +- Comments on queries. ### [v0.4](https://github.com/EverythingMe/redash/issues?milestone=3&state=open) From fbb84af9557e43d9518c48a558d922d68886dab4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 15:14:46 +0200 Subject: [PATCH 212/540] Update visualizations migration. --- migrations/create_visualizations.py | 80 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/migrations/create_visualizations.py b/migrations/create_visualizations.py index acb2e86ed..fc1847b42 100644 --- a/migrations/create_visualizations.py +++ b/migrations/create_visualizations.py @@ -1,66 +1,64 @@ import json +import peewee +from playhouse.migrate import Migrator +from redash import db from redash import models -# first run: - -# CREATE TABLE "visualizations" ( -# "id" serial NOT NULL PRIMARY KEY, -# "type" varchar(100) NOT NULL, -# "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED, -# "name" varchar(255) NOT NULL, -# "description" varchar(4096), -# "options" text NOT NULL -# ) -# ; - -# ALTER TABLE widgets ADD COLUMN "visualization_id" integer REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED; - if __name__ == '__main__': default_options = {"series": {"type": "bar"}} - # create 'table' visualization for all queries - print 'creating TABLE visualizations ...' - for query in Query.objects.all(): - vis = Visualization(query=query, name="Table", - description=query.description, - type="TABLE", options="{}") + db.connect_db() + + if not models.Visualization.table_exists(): + print "Creating visualization table..." + models.Visualization.create_table() + + with db.database.transaction(): + migrator = Migrator(db.database) + print "Adding visualization_id to widgets:" + migrator.add_column(models.Widget, models.Widget.visualization, 'visualization_id') + + print 'Creating TABLE visualizations for all queries...' + for query in models.Query.select(): + vis = models.Visualization(query=query, name="Table", + description=query.description, + type="TABLE", options="{}") vis.save() - - # create 'cohort' visualization for all queries named with 'cohort' - print 'creating COHORT visualizations ...' - for query in Query.objects.filter(name__icontains="cohort"): - vis = Visualization(query=query, name="Cohort", - description=query.description, - type="COHORT", options="{}") + print 'Creating COHORT visualizations for all queries named like %cohort%...' + for query in models.Query.select().where(peewee.fn.Icontains(models.Query.name)=="cohort"): + vis = models.Visualization(query=query, name="Cohort", + description=query.description, + type="COHORT", options="{}") vis.save() - - # create visualization for every widget (unless it already exists) - print 'migrating Widgets -> Visualizations ...' - for widget in Widget.objects.all(): - print 'processing widget %d:' % widget.id, + print 'Create visualization for all widgets (unless exists already):' + for widget in models.Widget.select(): + print 'Processing widget id: %d:' % widget.id query = widget.query vis_type = widget.type.upper() + if vis_type == 'GRID': + vis_type = 'TABLE' - vis = query.visualizations.filter(type=vis_type) + vis = query.visualizations.where(type=vis_type).first() if vis: - print 'visualization exists' + print '... visualization type (%s) found.' % vis_type widget.visualization = vis[0] widget.save() - else: - vis_name = widget.type.title() + vis_name = vis_type.title() options = json.loads(widget.options) vis_options = {"series": options} if options else default_options vis_options = json.dumps(vis_options) - vis = Visualization(query=query, name=vis_name, - description=query.description, - type=vis_type, options=vis_options) + vis = models.Visualization(query=query, name=vis_name, + description=query.description, + type=vis_type, options=vis_options) - print 'created visualization %s' % vis_type + print '... Created visualization for type: %s' % vis_type vis.save() widget.visualization = vis - widget.save() \ No newline at end of file + widget.save() + + db.close_db(None) \ No newline at end of file From 7bf391e772cbece78b521f1e357ced4bef6908f4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 15:20:58 +0200 Subject: [PATCH 213/540] Set automatic releases as 'prerelease'. --- bin/upload_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/upload_version.py b/bin/upload_version.py index ac9bd534f..3a074676d 100644 --- a/bin/upload_version.py +++ b/bin/upload_version.py @@ -15,7 +15,8 @@ if __name__ == '__main__': params = json.dumps({ 'tag_name': 'v{0}'.format(version), 'name': 're:dash v{0}'.format(version), - 'target_commitish': commit_sha + 'target_commitish': commit_sha, + 'prerelease': True }) response = requests.post('https://api.github.com/repos/everythingme/redash/releases', From c40cf2e7e8601d9231beda3e196df7dbc262eb09 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 16:40:26 +0200 Subject: [PATCH 214/540] Improve visualizations migration --- migrations/create_visualizations.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/migrations/create_visualizations.py b/migrations/create_visualizations.py index fc1847b42..3f743d75e 100644 --- a/migrations/create_visualizations.py +++ b/migrations/create_visualizations.py @@ -1,5 +1,4 @@ import json -import peewee from playhouse.migrate import Migrator from redash import db from redash import models @@ -16,34 +15,36 @@ if __name__ == '__main__': with db.database.transaction(): migrator = Migrator(db.database) print "Adding visualization_id to widgets:" + field = models.Widget.visualization + field.null = True migrator.add_column(models.Widget, models.Widget.visualization, 'visualization_id') print 'Creating TABLE visualizations for all queries...' for query in models.Query.select(): vis = models.Visualization(query=query, name="Table", - description=query.description, + description=query.description or "", type="TABLE", options="{}") vis.save() print 'Creating COHORT visualizations for all queries named like %cohort%...' - for query in models.Query.select().where(peewee.fn.Icontains(models.Query.name)=="cohort"): + for query in models.Query.select().where(models.Query.name ** "%cohort%"): vis = models.Visualization(query=query, name="Cohort", - description=query.description, + description=query.description or "", type="COHORT", options="{}") vis.save() print 'Create visualization for all widgets (unless exists already):' for widget in models.Widget.select(): print 'Processing widget id: %d:' % widget.id - query = widget.query vis_type = widget.type.upper() if vis_type == 'GRID': vis_type = 'TABLE' - vis = query.visualizations.where(type=vis_type).first() + query = models.Query.get_by_id(widget.query_id) + vis = query.visualizations.where(models.Visualization.type == vis_type).first() if vis: print '... visualization type (%s) found.' % vis_type - widget.visualization = vis[0] + widget.visualization = vis widget.save() else: vis_name = vis_type.title() @@ -53,7 +54,7 @@ if __name__ == '__main__': vis_options = json.dumps(vis_options) vis = models.Visualization(query=query, name=vis_name, - description=query.description, + description=query.description or "", type=vis_type, options=vis_options) print '... Created visualization for type: %s' % vis_type @@ -61,4 +62,9 @@ if __name__ == '__main__': widget.visualization = vis widget.save() - db.close_db(None) \ No newline at end of file + with db.database.transaction(): + migrator = Migrator(db.database) + print "Setting visualization_id as not null..." + migrator.set_nullable(models.Widget, models.Widget.visualization, False) + + db.close_db(None) \ No newline at end of file From 29531a361cb5cc031fcf7259ab8bc39dc505cbac Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 16:46:32 +0200 Subject: [PATCH 215/540] Move version information into python package. --- Makefile | 4 ++-- manage.py | 7 +++++-- redash/__init__.py | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0fe935305..4e44ebb23 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ NAME=redash -VERSION=0.3 -FULL_VERSION=$(VERSION).$(CIRCLE_BUILD_NUM) +VERSION=`python ./manage.py version` +FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM) FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz deps: diff --git a/manage.py b/manage.py index 19605bcc1..52640c6f7 100755 --- a/manage.py +++ b/manage.py @@ -10,7 +10,7 @@ atfork.stdlib_fixer.fix_logging_module() import argparse import logging import time -from redash import settings, app, data_manager +from redash import settings, app, data_manager, __version__ def start_workers(): @@ -53,6 +53,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(title='command', dest='command') + subparsers.add_parser('version', help='display version') subparsers.add_parser('worker', help='start query execution workers') server_parser = subparsers.add_parser('server', help='start api server') server_parser.add_argument('--debug', @@ -71,7 +72,9 @@ if __name__ == '__main__': args = parser.parse_args() - if args.command == "worker": + if args.command == 'version': + print __version__ + elif args.command == "worker": start_workers() elif args.command == 'db': create_db(args.create_tables, args.drop_tables) diff --git a/redash/__init__.py b/redash/__init__.py index 69ca6a721..cd2d6308f 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -7,6 +7,7 @@ from flask_peewee.db import Database import redis from redash import settings, utils +__version__ = '0.3.1' app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, From 669b1d9a6380911666d0cc9f78a7f6a8f8017141 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 17:09:07 +0200 Subject: [PATCH 216/540] Switch to Flask-Script. --- README.md | 8 +++--- manage.py | 72 +++++++++++++++++++----------------------------- requirements.txt | 1 + 3 files changed, 34 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 93445f8c0..64811f376 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,10 @@ It's very likely that in the future we will switch to [D3.js](http://d3js.org/) 2. Make sure you have `Python` v2.7, `pip`, PostgreSQL and Redis installed. 3. Install Python requirements: `pip install -r requirements.txt`. 4. Make a copy of the examples settings file: `cp redash/settings_example.py redash/settings.py` and edit the relevant settings. -5. Create database: `./manage.py db --create-tables`. -6. Start the web server: `./manage.py server`. -7. Start the worker: `./manage.py worker`. -8. Open `http://localhost:8888/` and query away. +5. Create database: `./manage.py database create_tables`. +6. Start the web server: `./manage.py runserver`. +7. Start the worker: `./manage.py runworker`. +8. Open `http://localhost:5000/` and query away. **Need help setting re:dash or one of the dependencies up?** Ping @arikfr on the IRC #redash channel or send a message to the [mailing list](https://groups.google.com/forum/#!forum/redash-users), and he will gladly help. diff --git a/manage.py b/manage.py index 52640c6f7..fe0d961e2 100755 --- a/manage.py +++ b/manage.py @@ -7,13 +7,23 @@ atfork.monkeypatch_os_fork_functions() import atfork.stdlib_fixer atfork.stdlib_fixer.fix_logging_module() -import argparse import logging import time -from redash import settings, app, data_manager, __version__ +from redash import settings, app, db, models, data_manager, __version__ +from flask.ext.script import Manager +manager = Manager(app) +database_manager = Manager(help="Manages the database (create/drop tables).") + +@manager.command +def version(): + """Displays re:dash version.""" + print __version__ + +@manager.command +def runworkers(): + """Starts the re:dash query executors/workers.""" -def start_workers(): try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') @@ -34,49 +44,25 @@ def start_workers(): logging.warning("Exiting; waiting for threads") data_manager.stop_workers() +@manager.shell +def make_shell_context(): + return dict(app=app, db=db, models=models) -def start_server(port, debug): - app.run(debug=debug, port=port) - - -def create_db(create_tables, drop_tables): +@database_manager.command +def create_tables(): + """Creates the database tables.""" from redash.models import create_db - create_db(create_tables, drop_tables) + create_db(True, False) +@database_manager.command +def drop_tables(): + """Drop the database tables.""" + from redash.models import create_db + + create_db(False, True) + +manager.add_command("database", database_manager) if __name__ == '__main__': - channel = logging.StreamHandler() - logging.getLogger().addHandler(channel) - logging.getLogger().setLevel(settings.LOG_LEVEL) - - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(title='command', dest='command') - - subparsers.add_parser('version', help='display version') - subparsers.add_parser('worker', help='start query execution workers') - server_parser = subparsers.add_parser('server', help='start api server') - server_parser.add_argument('--debug', - action='store_true', - help='start in debug mode (code reload)') - server_parser.add_argument('--port', - default=8888, - help='port to bind to') - db_parser = subparsers.add_parser('db', help='database operations') - db_parser.add_argument('--create-tables', - action='store_true', - help='Create all tables') - db_parser.add_argument('--drop-tables', - action='store_true', - help='Drop all tables') - - args = parser.parse_args() - - if args.command == 'version': - print __version__ - elif args.command == "worker": - start_workers() - elif args.command == 'db': - create_db(args.create_tables, args.drop_tables) - elif args.command == 'server': - start_server(args.port, args.debug) \ No newline at end of file + manager.run() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d423444d6..d1a788f3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ six==1.5.2 sqlparse==0.1.8 wsgiref==0.1.2 wtf-peewee==0.2.2 +Flask-Script==0.6.6 From 21ff1d7482027c10690e6a04fd77e3a5dce1ea7f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 17:37:37 +0200 Subject: [PATCH 217/540] Change coveralls badge to point at master branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64811f376..7f22d1b5f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # [_re:dash_](https://github.com/everythingme/redash) ![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") -[![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=refactor_flask)](https://coveralls.io/r/EverythingMe/redash?branch=refactor_flask) +[![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=master)](https://coveralls.io/r/EverythingMe/redash?branch=master) **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. From 68ef489d8ceabac642de1d4fb2e822d855efca8a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 17:37:47 +0200 Subject: [PATCH 218/540] Add dev_requirements.txt file. --- dev_requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 dev_requirements.txt diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 000000000..0b1ca3c15 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,2 @@ +nose==1.3.0 +coverage==3.7.1 From 7fd094ba39bcc0316a919b306a8c187cfffebc8e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 18:50:28 +0200 Subject: [PATCH 219/540] Tests for HMAC authentication. --- redash/authentication.py | 20 ++++++++++++---- tests/__init__.py | 1 - tests/controllers.py | 52 ++++++++++++++++++++++++++++++++++++++-- tests/factories.py | 2 +- 4 files changed, 66 insertions(+), 9 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index 618e079da..a512d463d 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -9,6 +9,16 @@ import werkzeug.wrappers from redash import models, settings +def sign(key, path, expires): + if not key: + return None + + h = hmac.new(str(key), msg=path, digestmod=hashlib.sha1) + h.update(str(expires)) + + return h.hexdigest() + + class HMACAuthentication(object): def __init__(self, auth): self.auth = auth @@ -19,15 +29,15 @@ class HMACAuthentication(object): @functools.wraps(fn) def decorated(*args, **kwargs): signature = request.args.get('signature') - expires = int(request.args.get('expires') or 0) + expires = float(request.args.get('expires') or 0) query_id = request.view_args.get('query_id', None) - if signature and query_id and time.time() < expires: + # TODO: 3600 should be a setting + if signature and query_id and time.time() < expires <= time.time() + 3600: query = models.Query.get(models.Query.id == query_id) - h = hmac.new(str(query.api_key), msg=request.path, digestmod=hashlib.sha1) - h.update(str(expires)) + calculated_signature = sign(query.api_key, request.path, expires) - if query.api_key and signature == h.hexdigest(): + if query.api_key and signature == calculated_signature: return fn(*args, **kwargs) # Work around for flask-restful testing only for flask.wrappers.Resource instead of diff --git a/tests/__init__.py b/tests/__init__.py index 452449f60..737b02878 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -17,7 +17,6 @@ for model in redash.models.all_models: class BaseTestCase(TestCase): def setUp(self): - redash.models.create_db(True, True) def tearDown(self): diff --git a/tests/controllers.py b/tests/controllers.py index 1b3032315..0ddba3ea4 100644 --- a/tests/controllers.py +++ b/tests/controllers.py @@ -1,9 +1,12 @@ from contextlib import contextmanager import json +import time from tests import BaseTestCase -from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory +from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ + query_result_factory from redash import app, models from redash.utils import json_dumps +from redash.authentication import sign @contextmanager @@ -271,5 +274,50 @@ class JobAPITest(BaseTestCase, AuthenticationTestMixin): class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def setUp(self): + super(CsvQueryResultAPITest, self).setUp() self.paths = [] - super(CsvQueryResultAPITest, self).setUp() \ No newline at end of file + self.query_result = query_result_factory.create() + self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id) + + # TODO: factor out the HMAC authentication tests + + def signature(self, expires): + return sign(self.query_result.query.api_key, self.path, expires) + + def test_redirect_when_unauthenticated(self): + with app.test_client() as c: + rv = c.get(self.path) + self.assertEquals(rv.status_code, 302) + + def test_redirect_for_wrong_signature(self): + with app.test_client() as c: + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0}) + self.assertEquals(rv.status_code, 302) + + def test_redirect_for_correct_signature_and_wrong_expires(self): + with app.test_client() as c: + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0}) + self.assertEquals(rv.status_code, 302) + + def test_redirect_for_correct_signature_and_no_expires(self): + with app.test_client() as c: + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)}) + self.assertEquals(rv.status_code, 302) + + def test_redirect_for_correct_signature_and_expires_too_long(self): + with app.test_client() as c: + expires = time.time()+(10*3600) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) + self.assertEquals(rv.status_code, 302) + + def test_returns_200_for_correct_signature(self): + with app.test_client() as c: + expires = time.time()+3600 + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) + self.assertEquals(rv.status_code, 200) + + def test_returns_200_for_authenticated_user(self): + with app.test_client() as c, authenticated_user(c): + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)) + self.assertEquals(rv.status_code, 200) + diff --git a/tests/factories.py b/tests/factories.py index 3f48ff428..7c606daca 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -37,7 +37,7 @@ query_factory = ModelFactory(redash.models.Query, user='test@everything.me') query_result_factory = ModelFactory(redash.models.QueryResult, - data='{}', + data='{"columns":{}, "rows":[]}', runtime=1, retrieved_at=datetime.datetime.now(), query=query_factory.create, From cc980edc669d28f69dd4fdf7c2ec34a3a03b39de Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 19:07:56 +0200 Subject: [PATCH 220/540] Remove coveralls.io integration as it's breaking builds. --- README.md | 3 +-- circle.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7f22d1b5f..380953b49 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # [_re:dash_](https://github.com/everythingme/redash) -![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") -[![Coverage Status](https://coveralls.io/repos/EverythingMe/redash/badge.png?branch=master)](https://coveralls.io/r/EverythingMe/redash?branch=master) +![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status") **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns. diff --git a/circle.yml b/circle.yml index ce37e51af..0f0aa0a53 100644 --- a/circle.yml +++ b/circle.yml @@ -8,7 +8,7 @@ machine: dependencies: pre: - make deps - - pip install requests coverage nose coveralls + - pip install requests coverage nose - pip install -r requirements.txt - cp redash/settings_example.py redash/settings.py cache_directories: @@ -18,7 +18,6 @@ test: override: - make test post: - - coveralls - rm redash/settings.py - make pack deployment: From fa43ff13655c7cc0873f83351016230a5ecfbdf0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 19:34:43 +0200 Subject: [PATCH 221/540] Set default visualization description to ''. --- rd_ui/app/scripts/directives.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a1b85e91b..0113480d5 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -77,7 +77,7 @@ 'query_id': q.id, 'type': Visualization.prototype.TYPES.CHART, 'name': '', - 'description': q.description, + 'description': q.description || '', 'options': newOptions() }; } From 52758fa66e83c16b687ab76f5f2bf8147571ca39 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 19:38:24 +0200 Subject: [PATCH 222/540] Return query with visualizations when saving. --- redash/controllers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/controllers.py b/redash/controllers.py index 15d77c950..f62614071 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -209,7 +209,7 @@ class QueryAPI(BaseResource): query = models.Query.get_by_id(query_id) - return query.to_dict(with_result=False) + return query.to_dict(with_result=False, with_visualizations=True) def get(self, query_id): q = models.Query.get(models.Query.id == query_id) From a1a7ca8a0a52420c2f25492ed344b40288d56ffb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 19:38:41 +0200 Subject: [PATCH 223/540] Set Visualization.description to nullable. --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index d9378d63d..30b4c357a 100644 --- a/redash/models.py +++ b/redash/models.py @@ -185,7 +185,7 @@ class Visualization(BaseModel): type = peewee.CharField(max_length=100) query = peewee.ForeignKeyField(Query, related_name='visualizations') name = peewee.CharField(max_length=255) - description = peewee.CharField(max_length=4096) + description = peewee.CharField(max_length=4096, null=True) options = peewee.TextField() class Meta: From 1e11f8032a6994830214a3070304f729c922f5d2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:02:38 +0200 Subject: [PATCH 224/540] Set description of default table visualization to "". --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index 30b4c357a..15c57b043 100644 --- a/redash/models.py +++ b/redash/models.py @@ -64,7 +64,7 @@ class Query(BaseModel): def create_default_visualizations(self): table_visualization = Visualization(query=self, name="Table", - description=self.description, + description='', type="TABLE", options="{}") table_visualization.save() From 1af3fc1c9654469c1111a10105092c82289824a3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:02:58 +0200 Subject: [PATCH 225/540] After duplicating a query, put user back on table tab. --- rd_ui/app/scripts/controllers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 9ec09e6e7..5bd291085 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -112,6 +112,9 @@ } else { // TODO: replace this with a safer method $location.path($location.path().replace(oldId, q.id)).replace(); + + // Reset visualizations tab to table after duplicating a query: + $location.hash('table'); } } }, function(httpResponse) { From 4318468957b986e448cc758ed4da17fe0e0537f8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:03:32 +0200 Subject: [PATCH 226/540] There is no bar chart type -- it's column. --- migrations/create_visualizations.py | 2 +- rd_ui/app/scripts/directives.js | 2 +- rd_ui/app/scripts/services.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/create_visualizations.py b/migrations/create_visualizations.py index 3f743d75e..9eb2a64d6 100644 --- a/migrations/create_visualizations.py +++ b/migrations/create_visualizations.py @@ -4,7 +4,7 @@ from redash import db from redash import models if __name__ == '__main__': - default_options = {"series": {"type": "bar"}} + default_options = {"series": {"type": "column"}} db.connect_db() diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 0113480d5..fe95acf7e 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -63,7 +63,7 @@ }; scope.seriesTypes = { 'Line': Visualization.prototype.SERIES_TYPES.LINE, - 'Bar': Visualization.prototype.SERIES_TYPES.BAR, + 'Column': Visualization.prototype.SERIES_TYPES.BAR, 'Area': Visualization.prototype.SERIES_TYPES.AREA }; diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 092412b7b..b05f035f1 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -294,7 +294,7 @@ }, SERIES_TYPES: { 'LINE': 'line', - 'BAR': 'bar', + 'COLUMN': 'column', 'AREA': 'area' } }; From 64afd62a1f0c51840e5131481c31c83d75fef3d8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:17:01 +0200 Subject: [PATCH 227/540] Add scatter plot type. cc: @christophervalles --- rd_ui/app/scripts/directives.js | 5 +- rd_ui/app/scripts/ng-highchart.js | 114 +++++++++++++++++++----------- 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index fe95acf7e..de01420fa 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -63,8 +63,9 @@ }; scope.seriesTypes = { 'Line': Visualization.prototype.SERIES_TYPES.LINE, - 'Column': Visualization.prototype.SERIES_TYPES.BAR, - 'Area': Visualization.prototype.SERIES_TYPES.AREA + 'Column': Visualization.prototype.SERIES_TYPES.COLUMN, + 'Area': Visualization.prototype.SERIES_TYPES.AREA, + 'Scatter': 'scatter' }; if (!scope.vis) { diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 701a23137..86bd35907 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -1,37 +1,10 @@ -(function(){ +(function () { 'use strict'; var defaultOptions = { title: { "text": null }, - tooltip: { - valueDecimals: 2, - formatter: function () { - if (moment.isMoment(this.x)) { - var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', - pointsCount = this.points.length; - - $.each(this.points, function (i, point) { - s += '
        ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - - if (pointsCount > 1 && point.percentage) { - s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; - } - }); - } else { - var s = "" + this.points[0].key + ""; - $.each(this.points, function (i, point) { - s+= '
        ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - }); - } - - return s; - }, - shared: true - }, xAxis: { type: 'datetime' }, @@ -70,12 +43,62 @@ enabled: false }, plotOptions: { - "column": { - "stacking": "normal", - "pointPadding": 0, - "borderWidth": 1, - "groupPadding": 0, - "shadow": false + column: { + stacking: "normal", + pointPadding: 0, + borderWidth: 1, + groupPadding: 0, + shadow: false, + tooltip: { + valueDecimals: 2, + formatter: function () { + if (moment.isMoment(this.x)) { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; + + $.each(this.points, function (i, point) { + s += '
        ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; + } + }); + } else { + var s = "" + this.points[0].key + ""; + $.each(this.points, function (i, point) { + s += '
        ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + }); + } + + return s; + }, + shared: true + } + }, + line: { + dataLabels: { + enabled: true + } + }, + scatter: { + marker: { + radius: 5, + states: { + hover: { + enabled: true, + lineColor: 'rgb(100,100,100)' + } + } + }, + states: { + hover: { + marker: { + enabled: false + } + } + } } }, series: [] @@ -106,7 +129,7 @@ var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults); // Update when options change - scope.$watch('options', function(newOptions) { + scope.$watch('options', function (newOptions) { initChart(newOptions); }, true); @@ -118,12 +141,13 @@ scope.chart.showLoading(); } else { drawChart(); - }; + } + ; }, true); function initChart(options) { if (scope.chart) { - scope.chart.destroy(); + scope.chart.destroy(); } $.extend(true, chartOptions, options); @@ -133,23 +157,27 @@ } function drawChart() { - while(scope.chart.series.length > 0) { + while (scope.chart.series.length > 0) { scope.chart.series[0].remove(true); } // todo series.type - if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) { + if (_.some(scope.series[0].data, function (p) { + return angular.isString(p.x) + })) { scope.chart.xAxis[0].update({type: 'category'}); // We need to make sure that for each category, each series has a value. - var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')})); + var categories = _.union.apply(this, _.map(scope.series, function (s) { + return _.pluck(s.data, 'x') + })); - _.each(scope.series, function(s) { + _.each(scope.series, function (s) { // TODO: move this logic to Query#getChartData var yValues = _.groupBy(s.data, 'x'); - var newData = _.sortBy(_.map(categories, function(category) { + var newData = _.sortBy(_.map(categories, function (category) { return { name: category, y: yValues[category] && yValues[category][0].y @@ -164,7 +192,7 @@ scope.chart.counters.color = 0; - _.each(scope.series, function(s) { + _.each(scope.series, function (s) { // here we override the series with the visualization config var _s = $.extend(true, {}, s, chartOptions['series']); scope.chart.addSeries(_s); From 69c07a41e9aece22eed6c03604628afd1f46c5ad Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:28:37 +0200 Subject: [PATCH 228/540] Make tooltip work for all chart types. --- rd_ui/app/scripts/ng-highchart.js | 91 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 86bd35907..50fbe4ea3 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -13,6 +13,37 @@ text: null } }, + tooltip: { + valueDecimals: 2, + formatter: function () { + if (!this.points) { + this.points = [this.point]; + }; + + if (moment.isMoment(this.x)) { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; + + $.each(this.points, function (i, point) { + s += '
        ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; + } + }); + } else { + var s = "" + this.points[0].key + ""; + $.each(this.points, function (i, point) { + s += '
        ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + }); + } + + return s; + }, + shared: true + }, exporting: { chartOptions: { title: { @@ -43,43 +74,34 @@ enabled: false }, plotOptions: { + area: { + marker: { + enabled: false, + symbol: 'circle', + radius: 2, + states: { + hover: { + enabled: true + } + } + } + }, column: { stacking: "normal", pointPadding: 0, borderWidth: 1, groupPadding: 0, - shadow: false, - tooltip: { - valueDecimals: 2, - formatter: function () { - if (moment.isMoment(this.x)) { - var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', - pointsCount = this.points.length; - - $.each(this.points, function (i, point) { - s += '
        ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - - if (pointsCount > 1 && point.percentage) { - s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; - } - }); - } else { - var s = "" + this.points[0].key + ""; - $.each(this.points, function (i, point) { - s += '
        ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - }); - } - - return s; - }, - shared: true - } + shadow: false }, line: { - dataLabels: { - enabled: true + marker: { + radius: 3, + }, + lineWidth: 1, + states: { + hover: { + lineWidth: 2 + } } }, scatter: { @@ -92,12 +114,9 @@ } } }, - states: { - hover: { - marker: { - enabled: false - } - } + tooltip: { + headerFormat: '{series.name}
        ', + pointFormat: '{point.x}, {point.y}' } } }, From 009726c62d74789eeef62a93b94d6ea66d0542d2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 20:42:01 +0200 Subject: [PATCH 229/540] Fix for high charts bug with stacked areas. --- rd_ui/app/scripts/ng-highchart.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 50fbe4ea3..3bc43dbcc 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -19,7 +19,7 @@ if (!this.points) { this.points = [this.point]; }; - + if (moment.isMoment(this.x)) { var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', pointsCount = this.points.length; @@ -214,8 +214,24 @@ _.each(scope.series, function (s) { // here we override the series with the visualization config var _s = $.extend(true, {}, s, chartOptions['series']); + + if (_s.type == 'area') { + _.each(_s.data, function (p) { + // This is an insane hack: somewhere deep in HighChart's code, + // when you stack areas, it tries to convert the string representation + // of point's x into a number. With the default implementation of toString + // it fails.... + + if (moment.isMoment(p.x)) { + p.x.toString = function () { + return String(this.toDate().getTime()); + }; + } + }); + }; + scope.chart.addSeries(_s); - }) + }); scope.chart.redraw(); scope.chart.hideLoading(); From 373b9c6a97d20396e393da6208ce18d358398517 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Feb 2014 21:03:24 +0200 Subject: [PATCH 230/540] Bring back logging level setting --- manage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manage.py b/manage.py index fe0d961e2..7385e7b92 100755 --- a/manage.py +++ b/manage.py @@ -65,4 +65,8 @@ def drop_tables(): manager.add_command("database", database_manager) if __name__ == '__main__': + channel = logging.StreamHandler() + logging.getLogger().addHandler(channel) + logging.getLogger().setLevel(settings.LOG_LEVEL) + manager.run() \ No newline at end of file From 97df37536cb090d8f02c19bd0d2f4c0b82ec3534 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 10 Feb 2014 09:55:49 +0200 Subject: [PATCH 231/540] Remove SERIES_TYPES from Visualization. --- rd_ui/app/scripts/directives.js | 8 ++++---- rd_ui/app/scripts/services.js | 5 ----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index de01420fa..abc7be701 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -62,9 +62,9 @@ 'Cohort': Visualization.prototype.TYPES.COHORT }; scope.seriesTypes = { - 'Line': Visualization.prototype.SERIES_TYPES.LINE, - 'Column': Visualization.prototype.SERIES_TYPES.COLUMN, - 'Area': Visualization.prototype.SERIES_TYPES.AREA, + 'Line': 'line', + 'Column': 'column', + 'Area': 'area', 'Scatter': 'scatter' }; @@ -94,7 +94,7 @@ // Chart return { 'series': { - 'type': Visualization.prototype.SERIES_TYPES.LINE + 'type': scope.seriesTypes[0] } }; } diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index b05f035f1..7216c2d2b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -291,11 +291,6 @@ 'CHART': 'CHART', 'COHORT': 'COHORT', 'TABLE': 'TABLE' - }, - SERIES_TYPES: { - 'LINE': 'line', - 'COLUMN': 'column', - 'AREA': 'area' } }; From 8fbcd0c34dabd4c659f4d0e7614f8539ffe2da58 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 10 Feb 2014 10:05:56 +0200 Subject: [PATCH 232/540] Performance improvements for chart rendering: 1. Don't redraw when adding or removing a single series, but redraw at the end. 2. Use $timeout to postpone high charts rendering until DOM is ready. --- rd_ui/app/scripts/ng-highchart.js | 53 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 3bc43dbcc..4faecc676 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -147,27 +147,34 @@ var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults); - // Update when options change - scope.$watch('options', function (newOptions) { - initChart(newOptions); - }, true); + // $timeout makes sure that this function invoked after the DOM ready. When draw/init + // invoked after the DOM is ready, we see first an empty HighCharts objects and later + // they get filled up. Which gives the feeling that the charts loading faster (otherwise + // we stare at an empty screen until the HighCharts object is ready). + $timeout(function(){ + // Update when options change + scope.$watch('options', function (newOptions) { + initChart(newOptions); + }, true); - //Update when charts data changes - scope.$watch(function () { - return (scope.series && scope.series.length) || 0; - }, function (length) { - if (!length || length == 0) { - scope.chart.showLoading(); - } else { - drawChart(); - } - ; - }, true); + //Update when charts data changes + scope.$watch(function () { + // TODO: this might be an issue in case the series change, but they stay + // with the same length + return (scope.series && scope.series.length) || 0; + }, function (length) { + if (!length || length == 0) { + scope.chart.showLoading(); + } else { + drawChart(); + }; + }, true); + }); function initChart(options) { if (scope.chart) { scope.chart.destroy(); - } + }; $.extend(true, chartOptions, options); @@ -177,10 +184,8 @@ function drawChart() { while (scope.chart.series.length > 0) { - scope.chart.series[0].remove(true); - } - - // todo series.type + scope.chart.series[0].remove(false); + }; if (_.some(scope.series[0].data, function (p) { return angular.isString(p.x) @@ -213,10 +218,10 @@ _.each(scope.series, function (s) { // here we override the series with the visualization config - var _s = $.extend(true, {}, s, chartOptions['series']); + s = _.extend(s, chartOptions['series']); - if (_s.type == 'area') { - _.each(_s.data, function (p) { + if (s.type == 'area') { + _.each(s.data, function (p) { // This is an insane hack: somewhere deep in HighChart's code, // when you stack areas, it tries to convert the string representation // of point's x into a number. With the default implementation of toString @@ -230,7 +235,7 @@ }); }; - scope.chart.addSeries(_s); + scope.chart.addSeries(s, false); }); scope.chart.redraw(); From e0d46c3942cac59ece711380c7f8cd813f942f21 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 10 Feb 2014 19:38:23 +0200 Subject: [PATCH 233/540] When clicking on widget in dashboard, it should take to the correct visualization tab. --- rd_ui/app/scripts/controllers.js | 3 ++- rd_ui/app/views/dashboard.html | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 5bd291085..0a6eef304 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -20,8 +20,9 @@ }); }; - $scope.open = function(query) { + $scope.open = function(query, visualization) { $location.path('/queries/' + query.id); + $location.hash(visualization.id); } $scope.query = new Query($scope.widget.visualization.query); diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index c17800441..bfd5841d1 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -21,7 +21,7 @@
        -

        +

        @@ -37,7 +37,7 @@ tooltip-placement="bottom">Updated: - + From bc787efc865ffd07d1a0df44f017f84ba4cce858 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 10 Feb 2014 21:06:52 +0200 Subject: [PATCH 234/540] Show delete/edit/create new visualization only to query owner. This is a temporary solution until we have owners for visualizations. --- rd_ui/app/views/queryfiddle.html | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 8b4e60d6b..694034f41 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -56,9 +56,9 @@ - × + × - +

      @@ -70,12 +70,16 @@
      -
      +

      -

      +
      -
      +

      +
      +
      +

      +

      From 02d6567347a4ef06338e43c592ced44f7d799cec Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Feb 2014 11:38:01 +0200 Subject: [PATCH 235/540] Imrpove (?) line chart settings --- rd_ui/app/scripts/ng-highchart.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 4faecc676..f234bccf3 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -95,12 +95,15 @@ }, line: { marker: { - radius: 3, + radius: 1 }, - lineWidth: 1, + lineWidth: 2, states: { hover: { - lineWidth: 2 + lineWidth: 2, + marker: { + radius: 3 + } } } }, From aabc912862d9ad52e574cb76f83be1a84eb4bd43 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Feb 2014 11:38:34 +0200 Subject: [PATCH 236/540] Graphite query runner support --- redash/data/manager.py | 5 ++- redash/data/query_runner_graphite.py | 46 ++++++++++++++++++++++++++++ redash/data/worker.py | 7 +++-- redash/settings_example.py | 9 +++--- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 redash/data/query_runner_graphite.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 46e12f9ab..d787f2e1c 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -153,9 +153,12 @@ class Manager(object): if self.workers: return self.workers - if getattr(settings, 'CONNECTION_ADAPTER', None) == "mysql": + if connection_type == 'mysql': from redash.data import query_runner_mysql runner = query_runner_mysql.mysql(connection_string) + elif connection_type == 'graphite': + from redash.data import query_runner_graphite + runner = query_runner_graphite.graphite(connection_string) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_graphite.py b/redash/data/query_runner_graphite.py new file mode 100644 index 000000000..dc664e010 --- /dev/null +++ b/redash/data/query_runner_graphite.py @@ -0,0 +1,46 @@ +""" +QueryRunner for Graphite. +""" +import json +import datetime +import requests +from redash.utils import JSONEncoder + + +def graphite(connection_params): + def transform_result(response): + columns = [{'name': 'Time::x'}, {'name': 'value::y'}, {'name': 'name::series'}] + rows = [] + + for series in response.json(): + for values in series['datapoints']: + timestamp = datetime.datetime.fromtimestamp(int(values[1])) + rows.append({'Time::x': timestamp, 'name::series': series['target'], 'value::y': values[0]}) + + data = {'columns': columns, 'rows': rows} + return json.dumps(data, cls=JSONEncoder) + + def query_runner(query): + base_url = "%s/render?format=json&" % connection_params['url'] + url = "%s%s" % (base_url, "&".join(query.split("\n"))) + error = None + data = None + + try: + response = requests.get(url, auth=connection_params['auth'], + verify=connection_params['verify']) + + if response.status_code == 200: + data = transform_result(response) + else: + error = "Failed getting results (%d)" % response.status_code + + except Exception, ex: + data = None + error = ex.message + + return data, error + + query_runner.annotate_query = False + + return query_runner \ No newline at end of file diff --git a/redash/data/worker.py b/redash/data/worker.py index 88cc92a8e..1913baf88 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -234,8 +234,11 @@ class Worker(threading.Thread): start_time = time.time() self.set_title("running query %s" % job_id) - annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ - (pid, job.id, job.query_hash, job.priority, job.query) + if getattr(self.query_runner, 'annotate_query', True): + annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ + (pid, job.id, job.query_hash, job.priority, job.query) + else: + annotated_query = job.query # TODO: here's the part that needs to be forked, not all of the worker process... data, error = self.query_runner(annotated_query) diff --git a/redash/settings_example.py b/redash/settings_example.py index 1f49cdb99..27a81ec5f 100644 --- a/redash/settings_example.py +++ b/redash/settings_example.py @@ -3,11 +3,12 @@ Example settings module. You should make your own copy as settings.py and enter """ REDIS_URL = "redis://localhost:6379" -# Either "pg" or "mysql" +# "pg", "graphite" or "mysql" CONNECTION_ADAPTER = "pg" -# Connection string for the database that is used to run queries against -# -- example mysql CONNECTION_STRING = "Server=;User=;Pwd=;Database=" -# -- example pg CONNECTION_STRING = "user= password= host= port=5439 dbname=" +# Connection string for the database that is used to run queries against. Examples: +# -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" +# -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" +# -- graphite: CONNECTION_STRING = {'url': 'https://graphite.yourcompany.com', 'auth': ('user', 'password'), 'verify': True} CONNECTION_STRING = "user= password= host= port=5439 dbname=" # Connection settings for re:dash's own database (where we store the queries, results, etc) DATABASE_CONFIG = { From 9b7227a88bc7e77535e325a7f44dcb8963ff70d9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Feb 2014 11:42:37 +0200 Subject: [PATCH 237/540] Make the default newOptions apply to all but the chart vis --- rd_ui/app/scripts/directives.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index abc7be701..a4d1e3974 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -79,24 +79,22 @@ 'type': Visualization.prototype.TYPES.CHART, 'name': '', 'description': q.description || '', - 'options': newOptions() + 'options': newOptions(Visualization.prototype.TYPES.CHART) }; } }, true); } function newOptions(chartType) { - if (chartType === Visualization.prototype.TYPES.COHORT) { - // empty config at the moment - return {}; + if (chartType === Visualization.prototype.TYPES.CHART) { + return { + 'series': { + 'type': 'column' + } + }; } - // Chart - return { - 'series': { - 'type': scope.seriesTypes[0] - } - }; + return {}; } scope.$watch('vis.type', function(type) { From e632cf1c420814e6cb3a776ab96272ebfb80e9ac Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Feb 2014 16:30:23 +0200 Subject: [PATCH 238/540] Support for pie charts. --- rd_ui/app/scripts/directives.js | 3 ++- rd_ui/app/scripts/ng-highchart.js | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a4d1e3974..b61aedc13 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -65,7 +65,8 @@ 'Line': 'line', 'Column': 'column', 'Area': 'area', - 'Scatter': 'scatter' + 'Scatter': 'scatter', + 'Pie': 'pie' }; if (!scope.vis) { diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index f234bccf3..a3c1c4160 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -107,6 +107,16 @@ } } }, + pie: { + allowPointSelect: true, + cursor: 'pointer', + dataLabels: { + enabled: true, + color: '#000000', + connectorColor: '#000000', + format: '{point.name}: {point.percentage:.1f} %' + } + }, scatter: { marker: { radius: 5, From f8302ab65a3294e2ae11e7d66d40c687a1406bd6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Feb 2014 16:30:41 +0200 Subject: [PATCH 239/540] Better support for single series tooltips. --- rd_ui/app/scripts/ng-highchart.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index a3c1c4160..a5eb378d5 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -33,10 +33,20 @@ } }); } else { - var s = "" + this.points[0].key + ""; - $.each(this.points, function (i, point) { - s += '
      ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); + var points = this.points; + var name = points[0].key || points[0].name; + + var s = "" + name + ""; + + $.each(points, function (i, point) { + if (points.length > 1) { + s += '
      ' + point.series.name + ': ' + Highcharts.numberFormat(point.y); + } else { + s += ": " + Highcharts.numberFormat(point.y); + if (point.percentage < 100) { + s += ' (' +Highcharts.numberFormat(point.percentage) + '%)'; + } + } }); } From 8bbb485d5bf9e6a430dfcf4e773514f5f7479997 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:41:36 +0200 Subject: [PATCH 240/540] Rename test files to test_. --- tests/{controllers.py => test_controllers.py} | 0 tests/{models.py => test_models.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{controllers.py => test_controllers.py} (100%) rename tests/{models.py => test_models.py} (100%) diff --git a/tests/controllers.py b/tests/test_controllers.py similarity index 100% rename from tests/controllers.py rename to tests/test_controllers.py diff --git a/tests/models.py b/tests/test_models.py similarity index 100% rename from tests/models.py rename to tests/test_models.py From cd999278815c720a8536143469a40393d6d003cb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:42:32 +0200 Subject: [PATCH 241/540] Add Honcho (foreman alternative in Python) file(s). The reason we have both Procfile and Honchofile is to be able to run both the workers and the web server in a single dyno on Heroku. --- Honchofile | 2 ++ Procfile | 1 + requirements.txt | 1 + 3 files changed, 4 insertions(+) create mode 100644 Honchofile create mode 100644 Procfile diff --git a/Honchofile b/Honchofile new file mode 100644 index 000000000..3e1708b10 --- /dev/null +++ b/Honchofile @@ -0,0 +1,2 @@ +web: ./manage.py runserver -p $PORT +worker: ./manage.py runworkers diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..3e492fafb --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: honcho start -f Honchofile -p $PORT diff --git a/requirements.txt b/requirements.txt index d1a788f3f..478dbba78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,4 @@ sqlparse==0.1.8 wsgiref==0.1.2 wtf-peewee==0.2.2 Flask-Script==0.6.6 +honcho==0.5.0 From 83909a07fa450cc001da86af241cd01bd2b16ac1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:43:41 +0200 Subject: [PATCH 242/540] Read settings from environment variables instead of a settings file. This is mostly done to make it easier to run re:dash on Heroku but should be convenient in other platforms too. --- .gitignore | 2 +- redash/settings.py | 44 ++++++++++++++++++++++++++++++++++++++ redash/settings_example.py | 30 -------------------------- tests/test_settings.py | 27 +++++++++++++++++++++++ 4 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 redash/settings.py delete mode 100644 redash/settings_example.py create mode 100644 tests/test_settings.py diff --git a/.gitignore b/.gitignore index 78e0bae52..a625307c0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ .idea *.pyc .coverage -redash/settings.py rd_ui/dist .DS_Store @@ -10,3 +9,4 @@ rd_ui/dist .vagrant Berksfile.lock redash/dump.rdb +.env diff --git a/redash/settings.py b/redash/settings.py new file mode 100644 index 000000000..0897a7cdb --- /dev/null +++ b/redash/settings.py @@ -0,0 +1,44 @@ +import os +import urlparse + + +def parse_db_url(url): + url_parts = urlparse.urlparse(url) + connection = { + 'engine': 'peewee.PostgresqlDatabase', + } + + if url_parts.hostname and not url_parts.path: + connection['name'] = url_parts.hostname + else: + connection['name'] = url_parts.path[1:] + connection['host'] = url_parts.hostname + connection['port'] = url_parts.port + connection['user'] = url_parts.username + connection['password'] = url_parts.password + + return connection + +REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379") + +# "pg", "graphite" or "mysql" +CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") +# Connection string for the database that is used to run queries against. Examples: +# -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" +# -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" +# -- graphite: CONNECTION_STRING = {'url': 'https://graphite.yourcompany.com', 'auth': ('user', 'password'), 'verify': True} +CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") + +# Connection settings for re:dash's own database (where we store the queries, results, etc) +DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres")) + +# Google Apps domain to allow access from; any user with email in this Google Apps will be allowed +# access +GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") +# Email addresses of admin users (comma separated) +ADMINS = os.environ.get("REDASH_ADMINS", '').split(',') +STATIC_ASSETS_PATH = os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/") +WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) +COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") +LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") +ANALYTICS = os.environ.get("REDASH_ANALYTICS", "") \ No newline at end of file diff --git a/redash/settings_example.py b/redash/settings_example.py deleted file mode 100644 index 27a81ec5f..000000000 --- a/redash/settings_example.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Example settings module. You should make your own copy as settings.py and enter the real settings. -""" -REDIS_URL = "redis://localhost:6379" - -# "pg", "graphite" or "mysql" -CONNECTION_ADAPTER = "pg" -# Connection string for the database that is used to run queries against. Examples: -# -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" -# -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" -# -- graphite: CONNECTION_STRING = {'url': 'https://graphite.yourcompany.com', 'auth': ('user', 'password'), 'verify': True} -CONNECTION_STRING = "user= password= host= port=5439 dbname=" -# Connection settings for re:dash's own database (where we store the queries, results, etc) -DATABASE_CONFIG = { - 'name': 'postgres', - 'engine': 'peewee.PostgresqlDatabase', -} -# Google Apps domain to allow access from; any user with email in this Google Apps will be allowed -# access -GOOGLE_APPS_DOMAIN = "" -# Email addresses of specific users not from the above set Google Apps Domain, that you want to -# allow access to re:dash -ALLOWED_USERS = [] -# Email addresses of admin users -ADMINS = [] -STATIC_ASSETS_PATH = "../rd_ui/dist/" -WORKERS_COUNT = 2 -COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f" -LOG_LEVEL = "INFO" -ANALYTICS = "" \ No newline at end of file diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 000000000..90888bdc4 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,27 @@ +from redash import settings as settings +from unittest import TestCase + + +class TestDatabaseUrlParser(TestCase): + def test_only_database_name(self): + config = settings.parse_db_url("postgresql://postgres") + self.assertEquals(config['name'], 'postgres') + + def test_host_and_database_name(self): + config = settings.parse_db_url("postgresql://localhost/postgres") + self.assertEquals(config['name'], 'postgres') + self.assertEquals(config['host'], 'localhost') + + def test_host_with_port_and_database_name(self): + config = settings.parse_db_url("postgresql://localhost:5432/postgres") + self.assertEquals(config['name'], 'postgres') + self.assertEquals(config['host'], 'localhost') + self.assertEquals(config['port'], 5432) + + def test_full_url(self): + config = settings.parse_db_url("postgresql://user:pass@localhost:5432/postgres") + self.assertEquals(config['name'], 'postgres') + self.assertEquals(config['host'], 'localhost') + self.assertEquals(config['port'], 5432) + self.assertEquals(config['user'], 'user') + self.assertEquals(config['password'], 'pass') \ No newline at end of file From fe7c4f96aac580239545a28ca25e2672af8067da Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:52:19 +0200 Subject: [PATCH 243/540] Fix: allow passing relative path for assets. --- redash/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/redash/settings.py b/redash/settings.py index 0897a7cdb..10024cb65 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -19,6 +19,11 @@ def parse_db_url(url): return connection + +def fix_assets_path(path): + fullpath = os.path.join(os.path.dirname(__file__), path) + return fullpath + REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379") # "pg", "graphite" or "mysql" @@ -37,7 +42,7 @@ DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") # Email addresses of admin users (comma separated) ADMINS = os.environ.get("REDASH_ADMINS", '').split(',') -STATIC_ASSETS_PATH = os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/") +STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/")) WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") From 5c9fe407024dc0eeeace9673180fd34987037a3e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:52:36 +0200 Subject: [PATCH 244/540] Bump version. --- redash/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/__init__.py b/redash/__init__.py index cd2d6308f..6309e01ff 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -7,7 +7,7 @@ from flask_peewee.db import Database import redis from redash import settings, utils -__version__ = '0.3.1' +__version__ = '0.3.2' app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, From b704406164d3f8806c3df8da68868b42a93906d3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 20:53:32 +0200 Subject: [PATCH 245/540] Example .env file. --- .env.example | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..7e953fcc8 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +REDASH_CONNECTION_ADAPTER=pg +REDASH_CONNECTION_STRING="dbname=data" +REDASH_STATIC_ASSETS_PATH=../rd_ui/app/ +REDASH_GOOGLE_APPS_DOMAIN= +REDASH_ADMINS= +REDASH_WORKERS_COUNT=2 +REDASH_COOKIE_SECRET= +REDASH_DATABASE_URL='postgresql://rd' +REDASH_LOG_LEVEL = "INFO" \ No newline at end of file From 5ffaf1aeadb23fda162fbbad022cedde559a66c7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Feb 2014 21:37:56 +0200 Subject: [PATCH 246/540] Fix CircleCI configuration --- circle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/circle.yml b/circle.yml index 0f0aa0a53..62ac4b7fc 100644 --- a/circle.yml +++ b/circle.yml @@ -10,7 +10,6 @@ dependencies: - make deps - pip install requests coverage nose - pip install -r requirements.txt - - cp redash/settings_example.py redash/settings.py cache_directories: - rd_ui/node_modules/ - rd_ui/app/bower_components/ From da746d15a019bf63747d1ef72b217bced212fefa Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 13:08:48 +0200 Subject: [PATCH 247/540] Fix: when updating query text the hash should change. --- manage.py | 1 + redash/controllers.py | 5 ++--- redash/models.py | 8 ++++++++ tests/test_models.py | 17 +++++++++++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/manage.py b/manage.py index 7385e7b92..9627fd7a9 100755 --- a/manage.py +++ b/manage.py @@ -20,6 +20,7 @@ def version(): """Displays re:dash version.""" print __version__ + @manager.command def runworkers(): """Starts the re:dash query executors/workers.""" diff --git a/redash/controllers.py b/redash/controllers.py index f62614071..e1d62c000 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -204,9 +204,8 @@ class QueryAPI(BaseResource): if 'latest_query_data_id' in query_def: query_def['latest_query_data'] = query_def.pop('latest_query_data_id') - update = models.Query.update(**query_def).where(models.Query.id == query_id) - update.execute() - + models.Query.update_instance(query_id, **query_def) + query = models.Query.get_by_id(query_id) return query.to_dict(with_result=False, with_visualizations=True) diff --git a/redash/models.py b/redash/models.py index 15c57b043..833d230c5 100644 --- a/redash/models.py +++ b/redash/models.py @@ -115,6 +115,14 @@ LEFT OUTER JOIN """ return cls.raw(query) + @classmethod + def update_instance(cls, query_id, **kwargs): + if 'query' in kwargs: + kwargs['query_hash'] = utils.gen_query_hash(kwargs['query']) + + update = cls.update(**kwargs).where(cls.id == query_id) + return update.execute() + def save(self, *args, **kwargs): self.query_hash = utils.gen_query_hash(self.query) self._set_api_key() diff --git a/tests/test_models.py b/tests/test_models.py index a408b51ed..f6c4889b3 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,5 +1,6 @@ from tests import BaseTestCase -from factories import dashboard_factory +from redash import models +from factories import dashboard_factory, query_factory class DashboardTest(BaseTestCase): @@ -12,4 +13,16 @@ class DashboardTest(BaseTestCase): d3 = dashboard_factory.create() self.assertNotEquals(d1.slug, d3.slug) - self.assertNotEquals(d2.slug, d3.slug) \ No newline at end of file + self.assertNotEquals(d2.slug, d3.slug) + + +class QueryTest(BaseTestCase): + def test_changing_query_text_changes_hash(self): + q = query_factory.create() + + old_hash = q.query_hash + models.Query.update_instance(q.id, query="SELECT 2;") + + q = models.Query.get_by_id(q.id) + + self.assertNotEquals(old_hash, q.query_hash) \ No newline at end of file From c82433e6b43954f6e5bc93659a2c0f5d7c2a5ed0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 14:50:42 +0200 Subject: [PATCH 248/540] CirlceCI: no longer need to delete settings.py. --- circle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/circle.yml b/circle.yml index 62ac4b7fc..23c414e72 100644 --- a/circle.yml +++ b/circle.yml @@ -17,7 +17,6 @@ test: override: - make test post: - - rm redash/settings.py - make pack deployment: github: From 95a6bab8b58eae3997e1e161e74295235eae3d83 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 16:16:55 +0200 Subject: [PATCH 249/540] Allow user to set the stacking of the chart. --- rd_ui/app/scripts/directives.js | 40 +++++++++++++++++++++++-- rd_ui/app/views/edit_visualization.html | 3 ++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index b61aedc13..c5f96920f 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -69,6 +69,14 @@ 'Pie': 'pie' }; + scope.stackingOptions = { + "None": "none", + "Normal": "normal", + "Percent": "percent" + }; + + scope.stacking = "none"; + if (!scope.vis) { // create new visualization // wait for query to load to populate with defaults @@ -90,20 +98,46 @@ if (chartType === Visualization.prototype.TYPES.CHART) { return { 'series': { - 'type': 'column' + 'type': 'column', + 'stacking': null } }; - } + }; return {}; } + var chartOptionsUnwatch = null; + scope.$watch('vis.type', function(type) { // if not edited by user, set name to match type if (type && scope.vis && !scope.visForm.name.$dirty) { // poor man's titlecase scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase(); } + + if (type && type == Visualization.prototype.TYPES.CHART) { + if (scope.vis.options.series.stacking === null) { + scope.stacking = "none"; + } else if (scope.vis.options.series.stacking === undefined) { + scope.stacking = "normal"; + } else { + scope.stacking = scope.vis.options.series.stacking ; + } + + chartOptionsUnwatch = scope.$watch("stacking", function(stacking) { + if (stacking == "none") { + scope.vis.options.series.stacking = null; + } else { + scope.vis.options.series.stacking = stacking; + } + }); + } else { + if (chartOptionsUnwatch) { + chartOptionsUnwatch(); + chartOptionsUnwatch = null; + } + } }); scope.toggleAdvancedMode = function() { @@ -111,7 +145,7 @@ }; scope.typeChanged = function() { - scope.vis.options = newOptions(); + scope.vis.options = newOptions(scope.vis.type); }; scope.submit = function() { diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html index bcba4a02e..670817503 100644 --- a/rd_ui/app/views/edit_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -12,6 +12,9 @@
      + + +
      From a207b93d0d1cf4d8ecde17f8152313ef12b0f7b9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 19:06:19 +0200 Subject: [PATCH 250/540] Fix: allow queries.description to be null. --- migrations/change_queries_description_to_nullable.py | 12 ++++++++++++ redash/models.py | 2 +- tests/test_controllers.py | 1 - 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 migrations/change_queries_description_to_nullable.py diff --git a/migrations/change_queries_description_to_nullable.py b/migrations/change_queries_description_to_nullable.py new file mode 100644 index 000000000..bf77dee2d --- /dev/null +++ b/migrations/change_queries_description_to_nullable.py @@ -0,0 +1,12 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.set_nullable(models.Query, models.Query.description, True) + + db.close_db(None) \ No newline at end of file diff --git a/redash/models.py b/redash/models.py index 833d230c5..b691300ed 100644 --- a/redash/models.py +++ b/redash/models.py @@ -51,7 +51,7 @@ class Query(BaseModel): id = peewee.PrimaryKeyField() latest_query_data = peewee.ForeignKeyField(QueryResult, null=True) name = peewee.CharField(max_length=255) - description = peewee.CharField(max_length=4096) + description = peewee.CharField(max_length=4096, null=True) query = peewee.TextField() query_hash = peewee.CharField(max_length=32) api_key = peewee.CharField(max_length=40) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 0ddba3ea4..5cfebc2a6 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -185,7 +185,6 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): user = 'test@everything.me' query_data = { 'name': 'Testing', - 'description': 'Description', 'query': 'SELECT 1', 'ttl': 3600 } From e5a365ba4116dc13f10af16042aaa18256f54ace Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 20:04:28 +0200 Subject: [PATCH 251/540] Bring back the ability to set allowed external users & publicly open re:dash. --- redash/authentication.py | 62 ++++++++++++++++++++++++--------------- redash/settings.py | 1 + tests/test_controllers.py | 46 ++++++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 27 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index a512d463d..1a566d9bc 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -1,11 +1,10 @@ import functools import hashlib import hmac -from flask import request, make_response -from flask.ext.googleauth import GoogleFederated +from flask import current_app, request, make_response, g, redirect, url_for +from flask.ext.googleauth import GoogleAuth import time from werkzeug.contrib.fixers import ProxyFix -import werkzeug.wrappers from redash import models, settings @@ -23,36 +22,53 @@ class HMACAuthentication(object): def __init__(self, auth): self.auth = auth - def required(self, fn): - wrapped_fn = self.auth.required(fn) + @staticmethod + def api_key_authentication(): + signature = request.args.get('signature') + expires = float(request.args.get('expires') or 0) + query_id = request.view_args.get('query_id', None) + # TODO: 3600 should be a setting + if signature and query_id and time.time() < expires <= time.time() + 3600: + query = models.Query.get(models.Query.id == query_id) + calculated_signature = sign(query.api_key, request.path, expires) + + if query.api_key and signature == calculated_signature: + return True + + return False + + @staticmethod + def is_user_logged_in(): + return g.user is not None + + @staticmethod + def valid_user(): + email = g.user['email'] + if not settings.GOOGLE_APPS_DOMAIN: + return True + + return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN) + + def required(self, fn): @functools.wraps(fn) def decorated(*args, **kwargs): - signature = request.args.get('signature') - expires = float(request.args.get('expires') or 0) - query_id = request.view_args.get('query_id', None) + if self.is_user_logged_in() and self.valid_user(): + return fn(*args, **kwargs) - # TODO: 3600 should be a setting - if signature and query_id and time.time() < expires <= time.time() + 3600: - query = models.Query.get(models.Query.id == query_id) - calculated_signature = sign(query.api_key, request.path, expires) + if self.api_key_authentication(): + return fn(*args, **kwargs) - if query.api_key and signature == calculated_signature: - return fn(*args, **kwargs) - - # Work around for flask-restful testing only for flask.wrappers.Resource instead of - # werkzeug.wrappers.Response - resp = wrapped_fn(*args, **kwargs) - if isinstance(resp, werkzeug.wrappers.Response): - resp = make_response(resp) - - return resp + blueprint = current_app.extensions['googleauth'].blueprint + # The make_response call is a work around for flask-restful testing only for + # flask.wrappers.Resource instead of werkzeug.wrappers.Response + return make_response(redirect(url_for("%s.login" % blueprint.name, next=request.url))) return decorated def setup_authentication(app): - openid_auth = GoogleFederated(settings.GOOGLE_APPS_DOMAIN, app) + openid_auth = GoogleAuth(app) app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = settings.COOKIE_SECRET diff --git a/redash/settings.py b/redash/settings.py index 10024cb65..b324db180 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -42,6 +42,7 @@ DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") # Email addresses of admin users (comma separated) ADMINS = os.environ.get("REDASH_ADMINS", '').split(',') +ALLOWED_EXTERNAL_USERS = os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '').split(',') STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/")) WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 0ddba3ea4..1f6fff041 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -1,14 +1,17 @@ from contextlib import contextmanager import json import time +from unittest import TestCase from tests import BaseTestCase from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ query_result_factory -from redash import app, models +from redash import app, models, settings from redash.utils import json_dumps from redash.authentication import sign +settings.GOOGLE_APPS_DOMAIN = "example.com" + @contextmanager def authenticated_user(c, user='test@example.com', name='John Test'): with c.session_transaction() as sess: @@ -45,7 +48,42 @@ class AuthenticationTestMixin(): self.assertEquals(200, rv.status_code) -class PingTest(BaseTestCase): +class TestAuthentication(TestCase): + def test_redirects_for_nonsigned_in_user(self): + with app.test_client() as c: + rv = c.get("/") + self.assertEquals(302, rv.status_code) + + def test_returns_content_when_authenticated_with_correct_domain(self): + settings.GOOGLE_APPS_DOMAIN = "example.com" + with app.test_client() as c, authenticated_user(c, user="test@example.com"): + rv = c.get("/") + self.assertEquals(200, rv.status_code) + + def test_redirects_when_authenticated_with_wrong_domain(self): + settings.GOOGLE_APPS_DOMAIN = "example.com" + with app.test_client() as c, authenticated_user(c, user="test@not-example.com"): + rv = c.get("/") + self.assertEquals(302, rv.status_code) + + def test_returns_content_when_user_in_allowed_list(self): + settings.GOOGLE_APPS_DOMAIN = "example.com" + settings.ALLOWED_EXTERNAL_USERS = ["test@not-example.com"] + + with app.test_client() as c, authenticated_user(c, user="test@not-example.com"): + rv = c.get("/") + self.assertEquals(200, rv.status_code) + + def test_returns_content_when_google_apps_domain_empty(self): + settings.GOOGLE_APPS_DOMAIN = "" + settings.ALLOWED_EXTERNAL_USERS = [] + + with app.test_client() as c, authenticated_user(c, user="test@whatever.com"): + rv = c.get("/") + self.assertEquals(200, rv.status_code) + + +class PingTest(TestCase): def test_ping(self): with app.test_client() as c: rv = c.get('/ping') @@ -83,7 +121,7 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.status_code, 404) def test_create_new_dashboard(self): - user_email = 'test@everything.me' + user_email = 'test@example.com' with app.test_client() as c, authenticated_user(c, user=user_email): dashboard_name = 'Test Dashboard' rv = json_request(c.post, '/api/dashboards', data={'name': dashboard_name}) @@ -182,7 +220,7 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.json['name'], 'Testing') def test_create_query(self): - user = 'test@everything.me' + user = 'test@example.com' query_data = { 'name': 'Testing', 'description': 'Description', From 8ad2c2a59e5df4a3cdabab6fc2c34f9f6c06e225 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 20:13:08 +0200 Subject: [PATCH 252/540] If only domain specified and not external users, use federated login. --- redash/authentication.py | 4 ++++ redash/settings.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index 1a566d9bc..bc9989b2f 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -69,6 +69,10 @@ class HMACAuthentication(object): def setup_authentication(app): openid_auth = GoogleAuth(app) + # If we don't have a list of external users, we can use Google's federated login, which limits + # the domain with which you can sign in. + if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN: + openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = settings.COOKIE_SECRET diff --git a/redash/settings.py b/redash/settings.py index b324db180..f5df483b6 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -24,6 +24,14 @@ def fix_assets_path(path): fullpath = os.path.join(os.path.dirname(__file__), path) return fullpath + +def array_from_string(str): + array = str.split(',') + if "" in array: + array.remove("") + + return array + REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379") # "pg", "graphite" or "mysql" @@ -41,8 +49,8 @@ DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql # access GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") # Email addresses of admin users (comma separated) -ADMINS = os.environ.get("REDASH_ADMINS", '').split(',') -ALLOWED_EXTERNAL_USERS = os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '').split(',') +ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", '')) +ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '')) STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/")) WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") From 07caee1d120bfd07a1b46eb107ba2ce1e54d5056 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 13 Feb 2014 20:16:36 +0200 Subject: [PATCH 253/540] Procfile changes: 1. Renamed Honchofile -> Procfile.heroku and changed it to work better with Heroku. 2. Added Procfile.dev for development. --- Procfile | 2 +- Honchofile => Procfile.dev | 0 Procfile.heroku | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) rename Honchofile => Procfile.dev (100%) create mode 100644 Procfile.heroku diff --git a/Procfile b/Procfile index 3e492fafb..c8f052fbb 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: honcho start -f Honchofile -p $PORT +web: honcho start -f Procfile.heroku -p $PORT diff --git a/Honchofile b/Procfile.dev similarity index 100% rename from Honchofile rename to Procfile.dev diff --git a/Procfile.heroku b/Procfile.heroku new file mode 100644 index 000000000..d7f3aabc1 --- /dev/null +++ b/Procfile.heroku @@ -0,0 +1,2 @@ +web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r +worker: ./manage.py runworkers From d73dbdeee0860365527dc6c649984cd6284f06a7 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Fri, 14 Feb 2014 11:57:42 +0000 Subject: [PATCH 254/540] Adding .ruby-version to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a625307c0..bee8c23eb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ rd_ui/dist Berksfile.lock redash/dump.rdb .env +.ruby-version \ No newline at end of file From 252ae7455a01a18325dd6c58cce614e84fd09d1c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Feb 2014 17:19:32 +0200 Subject: [PATCH 255/540] Auto-refresh button for dashboards. --- rd_ui/app/scripts/controllers.js | 39 ++++++++++++++++++++++++++++---- rd_ui/app/views/dashboard.html | 2 ++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 0a6eef304..3b76fae21 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -1,8 +1,37 @@ (function () { - var DashboardCtrl = function ($scope, $routeParams, $http, Dashboard) { - $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { - $scope.$parent.pageTitle = dashboard.name; - }); + var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) { + $scope.refreshEnabled = false; + $scope.refreshRate = 60; + + var refreshDashboard = function() { + $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { + $scope.$parent.pageTitle = dashboard.name; + }); + + autoRefresh(); + }; + + var autoRefresh = function() { + if ($scope.refreshEnabled) { + $timeout(refreshDashboard, $scope.refreshRate); + }; + } + + $scope.triggerRefresh = function(){ + $scope.refreshEnabled = !$scope.refreshEnabled; + + if ($scope.refreshEnabled) { + var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) { + return widget.visualization.query.ttl; + }).visualization.query.ttl; + + $scope.refreshRate = _.max([120, refreshRate * 2])*1000; + + autoRefresh(); + } + }; + + refreshDashboard(); }; var WidgetCtrl = function ($scope, $http, $location, Query) { @@ -390,7 +419,7 @@ } angular.module('redash.controllers', []) - .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl]) + .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl]) diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index bfd5841d1..9ee76f7f7 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -4,6 +4,8 @@

      {{dashboard.name}} + + From 574f75b2932724bc5d5ba02deaf3f88c1c528eba Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Feb 2014 17:19:48 +0200 Subject: [PATCH 256/540] Option to set every minute refresh rate. --- rd_ui/app/scripts/controllers.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 3b76fae21..b3e99695b 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -172,6 +172,7 @@ $scope.refreshOptions = [ {value: -1, name: 'No Refresh'}, + {value: 60, name: 'Every minute'}, ] _.each(_.range(1, 13), function(i) { From 30b86ea781183c0c54f46f2011380218a70aafa2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Feb 2014 17:57:26 +0200 Subject: [PATCH 257/540] Only refresh widgets that have their query data updated. --- rd_ui/app/scripts/controllers.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index b3e99695b..19a3b59f5 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -2,18 +2,29 @@ var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) { $scope.refreshEnabled = false; $scope.refreshRate = 60; - - var refreshDashboard = function() { - $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { - $scope.$parent.pageTitle = dashboard.name; - }); - - autoRefresh(); - }; + $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { + $scope.$parent.pageTitle = dashboard.name; + }); var autoRefresh = function() { if ($scope.refreshEnabled) { - $timeout(refreshDashboard, $scope.refreshRate); + $timeout(function() { + Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { + var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id'); + + _.each($scope.dashboard.widgets, function(row) { + _.each(row, function(widget, i) { + var newWidget = newWidgets[widget.id]; + if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) { + row[i] = newWidget[0]; + } + }); + }); + + autoRefresh(); + }); + + }, $scope.refreshRate); }; } @@ -30,8 +41,6 @@ autoRefresh(); } }; - - refreshDashboard(); }; var WidgetCtrl = function ($scope, $http, $location, Query) { From 07455e5821d21c988c7c5fcda9345e99355eb4e7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Feb 2014 18:01:44 +0200 Subject: [PATCH 258/540] Use database number from redis url if available. --- redash/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/redash/__init__.py b/redash/__init__.py index 6309e01ff..28bbe7dd7 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -33,7 +33,11 @@ def json_representation(data, code, headers=None): redis_url = urlparse.urlparse(settings.REDIS_URL) -redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password) +if redis_url.path: + redis_db = redis_url.path[1] +else: + redis_db = 0 +redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password) from redash import data data_manager = data.Manager(redis_connection, db) From 9241a7c35d416bbac3458d2349e824bf2db74ab1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 18 Feb 2014 11:15:46 +0200 Subject: [PATCH 259/540] User model & migration (ref #17) --- migrations/create_users.py | 46 ++++++++++++++++++++++++++++++++++++++ redash/authentication.py | 18 ++++++++++++++- redash/models.py | 33 ++++++++++++++------------- 3 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 migrations/create_users.py diff --git a/migrations/create_users.py b/migrations/create_users.py new file mode 100644 index 000000000..4471ee0d6 --- /dev/null +++ b/migrations/create_users.py @@ -0,0 +1,46 @@ +import json +import itertools +from playhouse.migrate import Migrator +from redash import db, settings +from redash import models + +if __name__ == '__main__': + db.connect_db() + + if not models.User.table_exists(): + print "Creating user table..." + models.User.create_table() + + migrator = Migrator(db.database) + with db.database.transaction(): + print "Creating user field on dashboard and queries..." + migrator.rename_column(models.Query, '"user"', "user_email") + migrator.rename_column(models.Dashboard, '"user"', "user_email") + + models.Query.user.null = True + models.Dashboard.user.null = True + + migrator.add_column(models.Query, models.Query.user, "user_id") + migrator.add_column(models.Dashboard, models.Dashboard.user, "user_id") + + print "Creating user for all queries and dashboards..." + for obj in itertools.chain(models.Query.select(), models.Dashboard.select()): + # Some old databases might have queries with empty string as user email: + email = obj.user_email or settings.ADMINS[0] + email = email.split(',')[0] + + print ".. {} , {}, {}".format(type(obj), obj.id, email) + + try: + user = models.User.get(models.User.email == email) + except models.User.DoesNotExist: + is_admin = email in settings.ADMINS + user = models.User.create(email=email, name="", is_admin=is_admin) + + obj.user = user + obj.save() + + print "Set user_id to non null..." + with db.database.transaction(): + migrator.set_nullable(models.Query, models.Query.user, True) + migrator.set_nullable(models.Dashboard, models.Dashboard.user, True) \ No newline at end of file diff --git a/redash/authentication.py b/redash/authentication.py index bc9989b2f..0a3065dbd 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -2,7 +2,7 @@ import functools import hashlib import hmac from flask import current_app, request, make_response, g, redirect, url_for -from flask.ext.googleauth import GoogleAuth +from flask.ext.googleauth import GoogleAuth, login import time from werkzeug.contrib.fixers import ProxyFix from redash import models, settings @@ -67,6 +67,22 @@ class HMACAuthentication(object): return decorated +def create_user(_, user): + try: + u = models.User.get(models.User.email == user.email) + if u.name != user.name: + current_app.logger.debug("Updating user name (%r -> %r)", u.name, user.name) + u.name = user.name + u.save() + except models.User.DoesNotExist: + current_app.logger.debug("Creating user object (%r)", user.name) + u = models.User(name=user.name, email=user.email) + u.save() + + +login.connect(create_user) + + def setup_authentication(app): openid_auth = GoogleAuth(app) # If we don't have a list of external users, we can use Google's federated login, which limits diff --git a/redash/models.py b/redash/models.py index b691300ed..85f20af7e 100644 --- a/redash/models.py +++ b/redash/models.py @@ -7,21 +7,22 @@ import peewee from redash import db, utils -#class User(db.Model): -# id = db.Column(db.Integer, primary_key=True) -# name = db.Column(db.String(320)) -# email = db.Column(db.String(160), unique=True) -# -# def __repr__(self): -# return '' % (self.name, self.email) - - class BaseModel(db.Model): @classmethod def get_by_id(cls, model_id): return cls.get(cls.id == model_id) +class User(BaseModel): + id = peewee.PrimaryKeyField() + name = peewee.CharField(max_length=320) + email = peewee.CharField(max_length=320, index=True, unique=True) + is_admin = peewee.BooleanField(default=False) + + def __unicode__(self): + return '' % (self.name, self.email) + + class QueryResult(db.Model): id = peewee.PrimaryKeyField() query_hash = peewee.CharField(max_length=32, index=True) @@ -56,7 +57,8 @@ class Query(BaseModel): query_hash = peewee.CharField(max_length=32) api_key = peewee.CharField(max_length=40) ttl = peewee.IntegerField() - user = peewee.CharField(max_length=360) + user_email = peewee.CharField(max_length=360) + user = peewee.ForeignKeyField(User) created_at = peewee.DateTimeField(default=datetime.datetime.now) class Meta: @@ -77,7 +79,7 @@ class Query(BaseModel): 'query': self.query, 'query_hash': self.query_hash, 'ttl': self.ttl, - 'user': self.user, + 'user': self.user.email, 'api_key': self.api_key, 'created_at': self.created_at, } @@ -131,7 +133,7 @@ LEFT OUTER JOIN def _set_api_key(self): if not self.api_key: self.api_key = hashlib.sha1( - u''.join([str(time.time()), self.query, self.user, self.name])).hexdigest() + u''.join((str(time.time()), self.query, str(self._data['user']), self.name)).encode('utf-8')).hexdigest() def __unicode__(self): return unicode(self.id) @@ -141,7 +143,8 @@ class Dashboard(db.Model): id = peewee.PrimaryKeyField() slug = peewee.CharField(max_length=140, index=True) name = peewee.CharField(max_length=100) - user = peewee.CharField(max_length=360) + user_email = peewee.CharField(max_length=360) + user = peewee.ForeignKeyField(User) layout = peewee.TextField() is_archived = peewee.BooleanField(default=False, index=True) created_at = peewee.DateTimeField(default=datetime.datetime.now) @@ -164,7 +167,7 @@ class Dashboard(db.Model): 'id': self.id, 'slug': self.slug, 'name': self.name, - 'user': self.user, + 'user': self.user.email, 'layout': layout, 'widgets': widgets_layout } @@ -245,7 +248,7 @@ class Widget(db.Model): def __unicode__(self): return u"%s" % self.id -all_models = (QueryResult, Query, Dashboard, Visualization, Widget) +all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget) def create_db(create_tables, drop_tables): From 791f2e0b34e2d2273e4dfddc749fb98dbcf91192 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 22 Feb 2014 14:42:08 +0200 Subject: [PATCH 260/540] Use of user object (fix views, update migrations and some). --- migrations/create_users.py | 24 ++++++++---- rd_ui/app/index.html | 3 +- rd_ui/app/scripts/controllers.js | 6 +-- rd_ui/app/views/queryfiddle.html | 2 +- redash/__init__.py | 1 - redash/authentication.py | 3 ++ redash/controllers.py | 19 +++++++--- redash/models.py | 65 ++++++++++++++++++++------------ tests/factories.py | 22 ++++++++++- tests/test_controllers.py | 24 +++++++----- tests/test_models.py | 4 +- 11 files changed, 117 insertions(+), 56 deletions(-) diff --git a/migrations/create_users.py b/migrations/create_users.py index 4471ee0d6..403bff021 100644 --- a/migrations/create_users.py +++ b/migrations/create_users.py @@ -1,5 +1,6 @@ import json import itertools +import peewee from playhouse.migrate import Migrator from redash import db, settings from redash import models @@ -14,14 +15,21 @@ if __name__ == '__main__': migrator = Migrator(db.database) with db.database.transaction(): print "Creating user field on dashboard and queries..." - migrator.rename_column(models.Query, '"user"', "user_email") - migrator.rename_column(models.Dashboard, '"user"', "user_email") + try: + migrator.rename_column(models.Query, '"user"', "user_email") + migrator.rename_column(models.Dashboard, '"user"', "user_email") + except peewee.ProgrammingError: + print "Failed to rename user column -- assuming it already exists" + with db.database.transaction(): models.Query.user.null = True models.Dashboard.user.null = True - migrator.add_column(models.Query, models.Query.user, "user_id") - migrator.add_column(models.Dashboard, models.Dashboard.user, "user_id") + try: + migrator.add_column(models.Query, models.Query.user, "user_id") + migrator.add_column(models.Dashboard, models.Dashboard.user, "user_id") + except peewee.ProgrammingError: + print "Failed to create user_id column -- assuming it already exists" print "Creating user for all queries and dashboards..." for obj in itertools.chain(models.Query.select(), models.Dashboard.select()): @@ -35,12 +43,14 @@ if __name__ == '__main__': user = models.User.get(models.User.email == email) except models.User.DoesNotExist: is_admin = email in settings.ADMINS - user = models.User.create(email=email, name="", is_admin=is_admin) + user = models.User.create(email=email, name=email, is_admin=is_admin) obj.user = user obj.save() print "Set user_id to non null..." with db.database.transaction(): - migrator.set_nullable(models.Query, models.Query.user, True) - migrator.set_nullable(models.Dashboard, models.Dashboard.user, True) \ No newline at end of file + migrator.set_nullable(models.Query, models.Query.user, False) + migrator.set_nullable(models.Dashboard, models.Dashboard.user, False) + migrator.set_nullable(models.Query, models.Query.user_email, True) + migrator.set_nullable(models.Dashboard, models.Dashboard.user_email, True) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index a3964f749..004da8099 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -124,7 +124,8 @@ var currentUser = {{ user|safe }}; currentUser.canEdit = function(object) { - return object.user && (object.user.indexOf(currentUser.name) != -1); + var user_id = object.user_id || (object.user && object.user.id); + return user_id && (user_id == currentUser.id); }; {{ analytics|safe }} diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 19a3b59f5..b731a585b 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -303,9 +303,9 @@ } if ($scope.selectedTab.key == 'my') { - return query.user == currentUser.name && query.name != 'New Query'; + return query.user.id == currentUser.id && query.name != 'New Query'; } else if ($scope.selectedTab.key == 'drafts') { - return query.user == currentUser.name && query.name == 'New Query'; + return query.user.id == currentUser.id && query.name == 'New Query'; } return query.name != 'New Query'; @@ -330,7 +330,7 @@ }, { 'label': 'Created By', - 'map': 'user' + 'map': 'user.name' }, { 'label': 'Created At', diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 694034f41..e331dbce3 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -34,7 +34,7 @@ Query runtime: {{queryResult.getRuntime() | durationHumanize}} | Last update time: | Rows: {{queryResult.getData().length}} | - Created by: {{query.user}} + Created by: {{query.user.name}}
      Refresh query:

      diff --git a/redash/__init__.py b/redash/__init__.py index 28bbe7dd7..549dbc6b7 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -14,7 +14,6 @@ app = Flask(__name__, static_folder=settings.STATIC_ASSETS_PATH, static_path='/static') - api = Api(app) # configure our database diff --git a/redash/authentication.py b/redash/authentication.py index 0a3065dbd..48114941d 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -79,6 +79,9 @@ def create_user(_, user): u = models.User(name=user.name, email=user.email) u.save() + user['id'] = u.id + user['is_admin'] = u.is_admin + login.connect(create_user) diff --git a/redash/controllers.py b/redash/controllers.py index e1d62c000..cbdef3bd1 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -39,8 +39,10 @@ def index(anything=None): user = { 'gravatar_url': gravatar_url, - 'is_admin': g.user['email'] in settings.ADMINS, - 'name': g.user['email'] + 'is_admin': g.user['is_admin'], + 'id': g.user['id'], + 'name': g.user['name'], + 'email': g.user['email'] } return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) @@ -80,9 +82,16 @@ def format_sql_query(): class BaseResource(Resource): decorators = [auth.required] + def __init__(self, *args, **kwargs): + super(BaseResource, self).__init__(*args, **kwargs) + self._user = None + @property def current_user(self): - return g.user['email'] + if not self._user: + self._user = models.User(id=g.user['id'], email=g.user['email'], name=g.user['name'], + is_admin=g.user['is_admin']) + return self._user class DashboardListAPI(BaseResource): @@ -111,9 +120,9 @@ class DashboardAPI(BaseResource): return dashboard.to_dict(with_widgets=True) def post(self, dashboard_slug): - # TODO: either convert all requests to use slugs or ids dashboard_properties = request.get_json(force=True) - dashboard = models.Dashboard.get(models.Dashboard.id == dashboard_slug) + # TODO: either convert all requests to use slugs or ids + dashboard = models.Dashboard.get_by_id(dashboard_slug) dashboard.layout = dashboard_properties['layout'] dashboard.name = dashboard_properties['name'] dashboard.save() diff --git a/redash/models.py b/redash/models.py index 85f20af7e..e3cc0fa48 100644 --- a/redash/models.py +++ b/redash/models.py @@ -19,8 +19,19 @@ class User(BaseModel): email = peewee.CharField(max_length=320, index=True, unique=True) is_admin = peewee.BooleanField(default=False) + class Meta: + db_table = 'users' + + def to_dict(self): + return { + 'id': self.id, + 'name': self.name, + 'email': self.email, + 'is_admin': self.is_admin + } + def __unicode__(self): - return '' % (self.name, self.email) + return '%r, %r' % (self.name, self.email) class QueryResult(db.Model): @@ -57,7 +68,7 @@ class Query(BaseModel): query_hash = peewee.CharField(max_length=32) api_key = peewee.CharField(max_length=40) ttl = peewee.IntegerField() - user_email = peewee.CharField(max_length=360) + user_email = peewee.CharField(max_length=360, null=True) user = peewee.ForeignKeyField(User) created_at = peewee.DateTimeField(default=datetime.datetime.now) @@ -70,7 +81,7 @@ class Query(BaseModel): type="TABLE", options="{}") table_visualization.save() - def to_dict(self, with_result=True, with_stats=False, with_visualizations=False): + def to_dict(self, with_result=True, with_stats=False, with_visualizations=False, with_user=True): d = { 'id': self.id, 'latest_query_data_id': self._data.get('latest_query_data', None), @@ -79,11 +90,15 @@ class Query(BaseModel): 'query': self.query, 'query_hash': self.query_hash, 'ttl': self.ttl, - 'user': self.user.email, 'api_key': self.api_key, 'created_at': self.created_at, } + if with_user: + d['user'] = self.user.to_dict() + else: + d['user_id'] = self._data['user'] + if with_stats: d['avg_runtime'] = self.avg_runtime d['min_runtime'] = self.min_runtime @@ -102,20 +117,17 @@ class Query(BaseModel): @classmethod def all_queries(cls): - query = """SELECT queries.*, query_stats.* -FROM queries -LEFT OUTER JOIN - (SELECT qu.query_hash, - count(0) AS "times_retrieved", - avg(runtime) AS "avg_runtime", - min(runtime) AS "min_runtime", - max(runtime) AS "max_runtime", - max(retrieved_at) AS "last_retrieved_at" - FROM queries qu - JOIN query_results qr ON qu.query_hash=qr.query_hash - GROUP BY qu.query_hash) query_stats ON query_stats.query_hash = queries.query_hash - """ - return cls.raw(query) + q = Query.select(Query, User, + peewee.fn.Count(QueryResult.id).alias('times_retrieved'), + peewee.fn.Avg(QueryResult.runtime).alias('avg_runtime'), + peewee.fn.Min(QueryResult.runtime).alias('min_runtime'), + peewee.fn.Max(QueryResult.runtime).alias('max_runtime'), + peewee.fn.Max(QueryResult.retrieved_at).alias('last_retrieved_at'))\ + .join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER)\ + .switch(Query).join(User)\ + .group_by(Query.id, User.id) + + return q @classmethod def update_instance(cls, query_id, **kwargs): @@ -139,11 +151,11 @@ LEFT OUTER JOIN return unicode(self.id) -class Dashboard(db.Model): +class Dashboard(BaseModel): id = peewee.PrimaryKeyField() slug = peewee.CharField(max_length=140, index=True) name = peewee.CharField(max_length=100) - user_email = peewee.CharField(max_length=360) + user_email = peewee.CharField(max_length=360, null=True) user = peewee.ForeignKeyField(User) layout = peewee.TextField() is_archived = peewee.BooleanField(default=False, index=True) @@ -156,8 +168,13 @@ class Dashboard(db.Model): layout = json.loads(self.layout) if with_widgets: - widgets = Widget.select(Widget, Visualization, Query, QueryResult).\ - where(Widget.dashboard == self.id).join(Visualization).join(Query).join(QueryResult) + widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\ + .where(Widget.dashboard == self.id)\ + .join(Visualization)\ + .join(Query)\ + .join(User)\ + .switch(Query)\ + .join(QueryResult) widgets = {w.id: w.to_dict() for w in widgets} widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) else: @@ -167,14 +184,14 @@ class Dashboard(db.Model): 'id': self.id, 'slug': self.slug, 'name': self.name, - 'user': self.user.email, + 'user_id': self._data['user'], 'layout': layout, 'widgets': widgets_layout } @classmethod def get_by_slug(cls, slug): - return cls.get(cls.slug==slug) + return cls.get(cls.slug == slug) def save(self, *args, **kwargs): if not self.slug: diff --git a/tests/factories.py b/tests/factories.py index 7c606daca..be491d1ce 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -26,15 +26,33 @@ class ModelFactory(object): kwargs = self._get_kwargs(override_kwargs) return self.model.create(**kwargs) + +class Sequence(object): + def __init__(self, string): + self.sequence = 0 + self.string = string + + def __call__(self): + self.sequence += 1 + + return self.string.format(self.sequence) + + +user_factory = ModelFactory(redash.models.User, + name='John Doe', email=Sequence('test{}@example.com'), + is_admin=False) + + dashboard_factory = ModelFactory(redash.models.Dashboard, - name='test', user='test@everything.me', layout='[]') + name='test', user=user_factory.create, layout='[]') + query_factory = ModelFactory(redash.models.Query, name='New Query', description='', query='SELECT 1', ttl=-1, - user='test@everything.me') + user=user_factory.create) query_result_factory = ModelFactory(redash.models.QueryResult, data='{"columns":{}, "rows":[]}', diff --git a/tests/test_controllers.py b/tests/test_controllers.py index f5a9a525c..bd927b1fe 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -4,7 +4,7 @@ import time from unittest import TestCase from tests import BaseTestCase from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ - query_result_factory + query_result_factory, user_factory from redash import app, models, settings from redash.utils import json_dumps from redash.authentication import sign @@ -13,9 +13,13 @@ from redash.authentication import sign settings.GOOGLE_APPS_DOMAIN = "example.com" @contextmanager -def authenticated_user(c, user='test@example.com', name='John Test'): +def authenticated_user(c, user=None): + if not user: + user = user_factory.create() + with c.session_transaction() as sess: - sess['openid'] = {'email': user, 'name': name} + sess['openid'] = {'email': user.email, 'name': user.name, + 'id': user.id, 'is_admin': user.is_admin} yield @@ -48,7 +52,7 @@ class AuthenticationTestMixin(): self.assertEquals(200, rv.status_code) -class TestAuthentication(TestCase): +class TestAuthentication(BaseTestCase): def test_redirects_for_nonsigned_in_user(self): with app.test_client() as c: rv = c.get("/") @@ -56,13 +60,13 @@ class TestAuthentication(TestCase): def test_returns_content_when_authenticated_with_correct_domain(self): settings.GOOGLE_APPS_DOMAIN = "example.com" - with app.test_client() as c, authenticated_user(c, user="test@example.com"): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@example.com")): rv = c.get("/") self.assertEquals(200, rv.status_code) def test_redirects_when_authenticated_with_wrong_domain(self): settings.GOOGLE_APPS_DOMAIN = "example.com" - with app.test_client() as c, authenticated_user(c, user="test@not-example.com"): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): rv = c.get("/") self.assertEquals(302, rv.status_code) @@ -70,7 +74,7 @@ class TestAuthentication(TestCase): settings.GOOGLE_APPS_DOMAIN = "example.com" settings.ALLOWED_EXTERNAL_USERS = ["test@not-example.com"] - with app.test_client() as c, authenticated_user(c, user="test@not-example.com"): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): rv = c.get("/") self.assertEquals(200, rv.status_code) @@ -78,7 +82,7 @@ class TestAuthentication(TestCase): settings.GOOGLE_APPS_DOMAIN = "" settings.ALLOWED_EXTERNAL_USERS = [] - with app.test_client() as c, authenticated_user(c, user="test@whatever.com"): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@whatever.com")): rv = c.get("/") self.assertEquals(200, rv.status_code) @@ -122,7 +126,7 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): def test_create_new_dashboard(self): user_email = 'test@example.com' - with app.test_client() as c, authenticated_user(c, user=user_email): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email=user_email)): dashboard_name = 'Test Dashboard' rv = json_request(c.post, '/api/dashboards', data={'name': dashboard_name}) self.assertEquals(rv.status_code, 200) @@ -227,7 +231,7 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): 'ttl': 3600 } - with app.test_client() as c, authenticated_user(c, user=user): + with app.test_client() as c, authenticated_user(c, user=user_factory.create(email=user)): rv = json_request(c.post, '/api/queries', data=query_data) self.assertEquals(rv.status_code, 200) diff --git a/tests/test_models.py b/tests/test_models.py index f6c4889b3..171aba8b9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,10 +8,10 @@ class DashboardTest(BaseTestCase): d1 = dashboard_factory.create() self.assertEquals(d1.slug, 'test') - d2 = dashboard_factory.create() + d2 = dashboard_factory.create(user=d1.user) self.assertNotEquals(d1.slug, d2.slug) - d3 = dashboard_factory.create() + d3 = dashboard_factory.create(user=d1.user) self.assertNotEquals(d1.slug, d3.slug) self.assertNotEquals(d2.slug, d3.slug) From 766840de685436887378a2e52cb244a73fd8aef0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 22 Feb 2014 14:52:04 +0200 Subject: [PATCH 261/540] Fix tests --- tests/__init__.py | 2 +- tests/test_controllers.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 737b02878..f29714713 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,7 +2,7 @@ from unittest import TestCase from redash import settings, db, app import redash.models -# TODO: this isn't pretty... :-) +# TODO: this isn't pretty... settings.DATABASE_CONFIG = { 'name': 'circle_test', 'engine': 'peewee.PostgresqlDatabase', diff --git a/tests/test_controllers.py b/tests/test_controllers.py index bd927b1fe..4d5a540f6 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -125,13 +125,13 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.status_code, 404) def test_create_new_dashboard(self): - user_email = 'test@example.com' - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email=user_email)): + user = user_factory.create() + with app.test_client() as c, authenticated_user(c, user=user): dashboard_name = 'Test Dashboard' rv = json_request(c.post, '/api/dashboards', data={'name': dashboard_name}) self.assertEquals(rv.status_code, 200) self.assertEquals(rv.json['name'], 'Test Dashboard') - self.assertEquals(rv.json['user'], user_email) + self.assertEquals(rv.json['user_id'], user.id) self.assertEquals(rv.json['layout'], []) def test_update_dashboard(self): @@ -224,19 +224,19 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): self.assertEquals(rv.json['name'], 'Testing') def test_create_query(self): - user = 'test@example.com' + user = user_factory.create() query_data = { 'name': 'Testing', 'query': 'SELECT 1', 'ttl': 3600 } - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email=user)): + with app.test_client() as c, authenticated_user(c, user=user): rv = json_request(c.post, '/api/queries', data=query_data) self.assertEquals(rv.status_code, 200) self.assertDictContainsSubset(query_data, rv.json) - self.assertEquals(rv.json['user'], user) + self.assertEquals(rv.json['user']['id'], user.id) self.assertIsNotNone(rv.json['api_key']) self.assertIsNotNone(rv.json['query_hash']) From 2d0998a9955528f23f8d55e3067af581414f368f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 24 Feb 2014 14:40:47 +0200 Subject: [PATCH 262/540] Update Getting Started instructions. --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 380953b49..1dd045b8c 100644 --- a/README.md +++ b/README.md @@ -46,14 +46,8 @@ It's very likely that in the future we will switch to [D3.js](http://d3js.org/) ## Getting Started -1. Download the [latest release](https://github.com/everythingme/redash/releases). -2. Make sure you have `Python` v2.7, `pip`, PostgreSQL and Redis installed. -3. Install Python requirements: `pip install -r requirements.txt`. -4. Make a copy of the examples settings file: `cp redash/settings_example.py redash/settings.py` and edit the relevant settings. -5. Create database: `./manage.py database create_tables`. -6. Start the web server: `./manage.py runserver`. -7. Start the worker: `./manage.py runworker`. -8. Open `http://localhost:5000/` and query away. +* [Setting up re:dash on Heroku in 5 minutes](https://github.com/EverythingMe/redash/wiki/Setting-up-re:dash-on-Heroku-in-5-minutes) +* [Setting re:dash on your own server (Ubuntu)](https://github.com/EverythingMe/redash/wiki/Setting-re:dash-on-your-own-server-(Ubuntu)) **Need help setting re:dash or one of the dependencies up?** Ping @arikfr on the IRC #redash channel or send a message to the [mailing list](https://groups.google.com/forum/#!forum/redash-users), and he will gladly help. From b81c3ba614b0256eba368c2252088bb8e2a602f5 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Mon, 24 Feb 2014 16:44:08 +0200 Subject: [PATCH 263/540] Fixed MySQL Errors --- redash/data/query_runner_mysql.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/redash/data/query_runner_mysql.py b/redash/data/query_runner_mysql.py index de9b7d7bf..a479322e9 100644 --- a/redash/data/query_runner_mysql.py +++ b/redash/data/query_runner_mysql.py @@ -43,7 +43,10 @@ def mysql(connection_string): cursor.close() except MySQLdb.Error, e: json_data = None - error = e.message + error = e.args[1] + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None except Exception as e: raise sys.exc_info()[1], None, sys.exc_info()[2] finally: From cbd1cf7c259e595c1f92122fdea2d7bbc2e0885f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 25 Feb 2014 08:16:30 +0200 Subject: [PATCH 264/540] Make sure visualization don't overflow --- rd_ui/app/styles/redash.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 2da630180..7a0b72841 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -213,4 +213,8 @@ to add those CSS styles here. */ color: white; background-color: #FF8080; border-radius: 50%; +} + +visualization-renderer > div { + overflow: scroll; } \ No newline at end of file From a0662d53237a4bfbaa5d0aa58b8266b9666a19c7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 25 Feb 2014 08:17:16 +0200 Subject: [PATCH 265/540] Remove outdated vagrant file --- Berksfile | 3 --- Vagrantfile | 60 ----------------------------------------------------- 2 files changed, 63 deletions(-) delete mode 100644 Berksfile delete mode 100644 Vagrantfile diff --git a/Berksfile b/Berksfile deleted file mode 100644 index acc59cb36..000000000 --- a/Berksfile +++ /dev/null @@ -1,3 +0,0 @@ -cookbook 'apt' -cookbook 'postgresql' -cookbook 'redash', git: 'git@github.com:EverythingMe/chef-redash.git' diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 415006f58..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,60 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = '2' - -POSTGRES_PASSWORD = 'securepass' - -# Currently, chef postgress cookbook works with cleartext paswords, -# unless the password begins with 'md5' -# See https://github.com/hw-cookbooks/postgresql/issues/95 -require "digest/md5" -postgres_password_md5 = 'md5'+Digest::MD5.hexdigest(POSTGRES_PASSWORD+'postgres') - -# After starting the vagrant machine, the application is accessible via the URL -# http://localhost:9999 -HOST_PORT_TO_FORWARD_TO_REDASH = 9999 - -# Deploy direcly the code in parent dir; Don't download a release tarball -live_testing_deployment = true - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = 'ubuntu-precise-cloudimg-amd64' - config.vm.box_url = 'http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box' - - if config.respond_to? :cache - config.cache.auto_detect = true - end - - config.berkshelf.enabled = true - config.omnibus.chef_version = :latest - - config.vm.network 'forwarded_port', guest: 8888, host: HOST_PORT_TO_FORWARD_TO_REDASH - - if live_testing_deployment - config.vm.synced_folder "..", "/opt/redash" - end - - config.vm.provision :chef_solo do |chef| - # run apt-get update before anything else (specifically postgresql).. - chef.add_recipe 'apt' - chef.add_recipe 'redash::redis_for_redash' - chef.add_recipe 'postgresql::client' - chef.add_recipe 'postgresql::server' - chef.add_recipe 'redash::redash_pg_schema' - chef.add_recipe 'redash::redash' - # chef.log_level = :debug - chef.json = { - 'apt' => { 'compiletime' => true }, - 'postgresql' => { 'password' => {'postgres' => postgres_password_md5 } }, - 'redash' => { 'db' => {'host' => 'localhost', - 'user' => 'postgres', - 'password' => POSTGRES_PASSWORD }, - 'allow' => {'google_app_domain' => 'gmail.com', - 'admins' => ['joe@egmail.com','jack@gmail.com']}, - 'install_tarball' => !live_testing_deployment, - 'user' => live_testing_deployment ? 'vagrant' : 'redash'} - } - end -end From 6d14c5c55522a65756574823770246cf887fc62a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 25 Feb 2014 08:36:59 +0200 Subject: [PATCH 266/540] Fix graphite settings example --- redash/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/settings.py b/redash/settings.py index f5df483b6..c20032851 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -39,7 +39,7 @@ CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # Connection string for the database that is used to run queries against. Examples: # -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" # -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" -# -- graphite: CONNECTION_STRING = {'url': 'https://graphite.yourcompany.com', 'auth': ('user', 'password'), 'verify': True} +# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true} CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From edcf0661a6aee84b731d435164a2c29d20561530 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 25 Feb 2014 08:37:19 +0200 Subject: [PATCH 267/540] Fix: add parsing of graphite settings --- redash/data/manager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index d787f2e1c..5061dff0f 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -158,7 +158,12 @@ class Manager(object): runner = query_runner_mysql.mysql(connection_string) elif connection_type == 'graphite': from redash.data import query_runner_graphite - runner = query_runner_graphite.graphite(connection_string) + connection_params = json.loads(connection_string) + if connection_params['auth']: + connection_params['auth'] = tuple(connection_params['auth']) + else: + connection_params['auth'] = None + runner = query_runner_graphite.graphite(connection_params) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) From 8be95262d464e68f55e4cddde99ae0308a468cad Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Tue, 25 Feb 2014 18:14:47 +0000 Subject: [PATCH 268/540] DAT-741 --- rd_ui/app/scripts/controllers.js | 9 ++++-- rd_ui/app/scripts/directives.js | 47 +++++++++++++++++++++++-------- rd_ui/app/scripts/ng-highchart.js | 2 +- rd_ui/app/scripts/services.js | 2 +- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 0a6eef304..585764852 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -92,10 +92,15 @@ }) } - $scope.saveQuery = function (duplicate, oldId) { + $scope.saveQuery = function (duplicate, oldId, updateUrl) { if (!oldId) { oldId = $scope.query.id; } + + if (updateUrl == undefined) { + updateUrl = true; + } + delete $scope.query.latest_query_data; $scope.query.$save(function (q) { pristineHash = q.getHash(); @@ -108,7 +113,7 @@ } if (oldId != q.id) { - if (oldId == undefined) { + if (oldId == undefined && updateUrl) { $location.path($location.path().replace('new', q.id)).replace(); } else { // TODO: replace this with a safer method diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index c5f96920f..cb95758cc 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -46,7 +46,7 @@ } }]); - directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { + directives.directive('editVisulatizationForm', ['Visualization', 'growl', '$location', function(Visualization, growl, $location) { return { restrict: 'E', templateUrl: '/views/edit_visualization.html', @@ -76,7 +76,7 @@ }; scope.stacking = "none"; - + if (!scope.vis) { // create new visualization // wait for query to load to populate with defaults @@ -149,17 +149,42 @@ }; scope.submit = function() { + if (!scope.query.id) { + scope.$parent.saveQuery(false, null, false); + var unwatch = scope.$watch('query', function(q) { + if (q && q.id) { + unwatch(); + scope.vis['query_id'] = q.id; + + scope.saveVisualization(false); + + $location.path($location.path().replace('new', q.id)).replace(); + } + }, true); + } else { + scope.saveVisualization(); + } + }; + + scope.saveVisualization = function(updateTabs) { + if (updateTabs == undefined) { + updateTabs = true; + } + Visualization.save(scope.vis, function success(result) { growl.addSuccessMessage("Visualization saved"); - - scope.vis = result; - - var visIds = _.pluck(scope.query.visualizations, 'id'); - var index = visIds.indexOf(result.id); - if (index > -1) { - scope.query.visualizations[index] = result; - } else { - scope.query.visualizations.push(result); + + if (updateTabs) { + scope.vis = result; + + var visIds = _.pluck(scope.query.visualizations, 'id'); + var index = visIds.indexOf(result.id); + + if (index > -1) { + scope.query.visualizations[index] = result; + } else { + scope.query.visualizations.push(result); + } } }, function error() { growl.addErrorMessage("Visualization could not be saved"); diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index a5eb378d5..f7d06418c 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -210,7 +210,7 @@ scope.chart.series[0].remove(false); }; - if (_.some(scope.series[0].data, function (p) { + if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) { return angular.isString(p.x) })) { scope.chart.xAxis[0].update({type: 'category'}); diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 7216c2d2b..3693ca707 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -279,7 +279,7 @@ Query.prototype.getHash = function() { return [this.name, this.description, this.query].join('!#'); }; - + return Query; }; From 3c24e76eb4e8a533c38941ed8c75404f78444a9a Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Tue, 25 Feb 2014 19:59:14 +0000 Subject: [PATCH 269/540] UX/UI issues with visualizations fixed --- rd_ui/app/scripts/directives.js | 25 ++++++++++--------------- rd_ui/app/scripts/filters.js | 11 +++++++++++ rd_ui/app/views/edit_visualization.html | 2 +- rd_ui/app/views/queryfiddle.html | 2 +- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index cb95758cc..17156b2ef 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -83,17 +83,16 @@ var unwatch = scope.$watch('query', function(q) { if (q && q.id) { unwatch(); - scope.vis = { - 'query_id': q.id, - 'type': Visualization.prototype.TYPES.CHART, - 'name': '', - 'description': q.description || '', - 'options': newOptions(Visualization.prototype.TYPES.CHART) - }; + + if (!scope.vis) { + scope.vis = { + 'query_id': q.id, + }; + } } }, true); } - + function newOptions(chartType) { if (chartType === Visualization.prototype.TYPES.CHART) { return { @@ -110,12 +109,6 @@ var chartOptionsUnwatch = null; scope.$watch('vis.type', function(type) { - // if not edited by user, set name to match type - if (type && scope.vis && !scope.visForm.name.$dirty) { - // poor man's titlecase - scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase(); - } - if (type && type == Visualization.prototype.TYPES.CHART) { if (scope.vis.options.series.stacking === null) { scope.stacking = "none"; @@ -175,7 +168,9 @@ growl.addSuccessMessage("Visualization saved"); if (updateTabs) { - scope.vis = result; + scope.vis = { + 'query_id': scope.query.id + }; var visIds = _.pluck(scope.query.visualizations, 'id'); var index = visIds.indexOf(result.id); diff --git a/rd_ui/app/scripts/filters.js b/rd_ui/app/scripts/filters.js index baf5618d6..0ecb0e4e6 100644 --- a/rd_ui/app/scripts/filters.js +++ b/rd_ui/app/scripts/filters.js @@ -47,4 +47,15 @@ angular.module('redash.filters', []). } return 12; } + }) + + .filter('capitalize', function () { + return function (text) { + if (text) { + return text[0].toUpperCase() + text.slice(1).toLowerCase(); + } else { + return null; + } + + } }); \ No newline at end of file diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html index 670817503..2521902f3 100644 --- a/rd_ui/app/views/edit_visualization.html +++ b/rd_ui/app/views/edit_visualization.html @@ -1,7 +1,7 @@
      - +
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 694034f41..372bebc72 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -58,7 +58,7 @@ × - +
      From 51a37cae3d344d7f2140d2d32c5914858120a905 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 27 Feb 2014 10:24:28 +0200 Subject: [PATCH 270/540] Fix: saving new query. --- rd_ui/app/scripts/controllers.js | 2 +- redash/controllers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index b731a585b..38405487d 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -239,7 +239,7 @@ $scope.queryResult = $scope.query.getQueryResult(); }); } else { - $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser.name}); + $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser}); $scope.lockButton(false); } diff --git a/redash/controllers.py b/redash/controllers.py index cbdef3bd1..e91f73409 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -207,7 +207,7 @@ class QueryListAPI(BaseResource): class QueryAPI(BaseResource): def post(self, query_id): query_def = request.get_json(force=True) - for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: + for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']: query_def.pop(field, None) if 'latest_query_data_id' in query_def: From 5babab85c8f20744b0a53d3b65feb584b29013b9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 27 Feb 2014 12:45:56 +0200 Subject: [PATCH 271/540] Remove milestone v0.2 from README. --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 1dd045b8c..b37a9992a 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,6 @@ It's very likely that in the future we will switch to [D3.js](http://d3js.org/) Below you can see the "big" features of the next 3 releases (for full list, click on the link): -### [v0.2](https://github.com/EverythingMe/redash/issues?milestone=1&state=open) - -- Ability to generate multiple visualizations for a single query (dataset) in a more flexible way than today. Also easier extensbility points to add additional visualizations. -- Support for API access using API keys, instead of Google Login. -- UI Improvements (better notifications & flows, improved queries page) - ### [v0.3](https://github.com/EverythingMe/redash/issues?milestone=2&state=open) - Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors). From 2a3bcc2ecb3bc5bde0bb8f88c9f0019a76bdf6aa Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 27 Feb 2014 12:55:06 +0200 Subject: [PATCH 272/540] Bump version. --- redash/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 549dbc6b7..a949a2d8b 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -7,7 +7,7 @@ from flask_peewee.db import Database import redis from redash import settings, utils -__version__ = '0.3.2' +__version__ = '0.3.3' app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, @@ -41,4 +41,4 @@ redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.por from redash import data data_manager = data.Manager(redis_connection, db) -from redash import controllers \ No newline at end of file +from redash import controllers From 269cbe839b7b5e6ff909e0997a7beef092ce40e3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 14:42:13 +0200 Subject: [PATCH 273/540] Add flask_login and use it for managing authentication --- migrations/add_password_to_users.py | 12 +++++++ redash/authentication.py | 51 +++++++++++++++----------- redash/controllers.py | 22 +++++------- redash/models.py | 11 +++++- requirements.txt | 2 ++ tests/test_controllers.py | 55 +++++++++++++++-------------- 6 files changed, 90 insertions(+), 63 deletions(-) create mode 100644 migrations/add_password_to_users.py diff --git a/migrations/add_password_to_users.py b/migrations/add_password_to_users.py new file mode 100644 index 000000000..3e86e9114 --- /dev/null +++ b/migrations/add_password_to_users.py @@ -0,0 +1,12 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.add_column(models.User, models.User.password_hash, 'password_hash') + + db.close_db(None) diff --git a/redash/authentication.py b/redash/authentication.py index 48114941d..e40cf8729 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -3,10 +3,13 @@ import hashlib import hmac from flask import current_app, request, make_response, g, redirect, url_for from flask.ext.googleauth import GoogleAuth, login +from flask.ext.login import LoginManager, login_user, current_user import time from werkzeug.contrib.fixers import ProxyFix from redash import models, settings +login_manager = LoginManager() + def sign(key, path, expires): if not key: @@ -40,20 +43,12 @@ class HMACAuthentication(object): @staticmethod def is_user_logged_in(): - return g.user is not None - - @staticmethod - def valid_user(): - email = g.user['email'] - if not settings.GOOGLE_APPS_DOMAIN: - return True - - return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN) + return current_user.is_authenticated() def required(self, fn): @functools.wraps(fn) def decorated(*args, **kwargs): - if self.is_user_logged_in() and self.valid_user(): + if self.is_user_logged_in(): return fn(*args, **kwargs) if self.api_key_authentication(): @@ -67,23 +62,35 @@ class HMACAuthentication(object): return decorated -def create_user(_, user): +def validate_email(email): + if not settings.GOOGLE_APPS_DOMAIN: + return True + + return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN) + + +def create_and_login_user(_, openid_user): + if not validate_email(openid_user.email): + return + try: - u = models.User.get(models.User.email == user.email) - if u.name != user.name: - current_app.logger.debug("Updating user name (%r -> %r)", u.name, user.name) - u.name = user.name - u.save() + user = models.User.get(models.User.email == openid_user.email) + if user.name != openid_user.name: + current_app.logger.debug("Updating user name (%r -> %r)", user.name, openid_user.name) + user.name = openid_user.name + user.save() except models.User.DoesNotExist: current_app.logger.debug("Creating user object (%r)", user.name) - u = models.User(name=user.name, email=user.email) - u.save() + user = models.User.create(name=openid_user.name, email=openid_user.email) - user['id'] = u.id - user['is_admin'] = u.is_admin + login_user(user, remember=True) + +login.connect(create_and_login_user) -login.connect(create_user) +@login_manager.user_loader +def load_user(user_id): + return models.User.select().where(models.User.id == user_id).first() def setup_authentication(app): @@ -92,6 +99,8 @@ def setup_authentication(app): # the domain with which you can sign in. if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN: openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN + + login_manager.init_app(app) app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = settings.COOKIE_SECRET diff --git a/redash/controllers.py b/redash/controllers.py index e91f73409..ff50d441b 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -13,6 +13,7 @@ import datetime from flask import g, render_template, send_from_directory, make_response, request, jsonify from flask.ext.restful import Resource, abort +from flask_login import current_user import sqlparse from redash import settings, utils @@ -34,15 +35,15 @@ def ping(): @app.route('/') @auth.required def index(anything=None): - email_md5 = hashlib.md5(g.user['email'].lower()).hexdigest() + email_md5 = hashlib.md5(current_user.email.lower()).hexdigest() gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5 user = { 'gravatar_url': gravatar_url, - 'is_admin': g.user['is_admin'], - 'id': g.user['id'], - 'name': g.user['name'], - 'email': g.user['email'] + 'is_admin': current_user.is_admin, + 'id': current_user.id, + 'name': current_user.name, + 'email': current_user.email } return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) @@ -86,13 +87,6 @@ class BaseResource(Resource): super(BaseResource, self).__init__(*args, **kwargs) self._user = None - @property - def current_user(self): - if not self._user: - self._user = models.User(id=g.user['id'], email=g.user['email'], name=g.user['name'], - is_admin=g.user['is_admin']) - return self._user - class DashboardListAPI(BaseResource): def get(self): @@ -104,7 +98,7 @@ class DashboardListAPI(BaseResource): def post(self): dashboard_properties = request.get_json(force=True) dashboard = models.Dashboard(name=dashboard_properties['name'], - user=self.current_user, + user=current_user.id, layout='[]') dashboard.save() return dashboard.to_dict() @@ -192,7 +186,7 @@ class QueryListAPI(BaseResource): for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: query_def.pop(field, None) - query_def['user'] = self.current_user + query_def['user'] = current_user.id query = models.Query(**query_def) query.save() diff --git a/redash/models.py b/redash/models.py index e3cc0fa48..a79bf442b 100644 --- a/redash/models.py +++ b/redash/models.py @@ -3,6 +3,8 @@ import hashlib import time import datetime from flask.ext.peewee.utils import slugify +from flask.ext.login import UserMixin +from passlib.apps import custom_app_context as pwd_context import peewee from redash import db, utils @@ -13,10 +15,11 @@ class BaseModel(db.Model): return cls.get(cls.id == model_id) -class User(BaseModel): +class User(BaseModel, UserMixin): id = peewee.PrimaryKeyField() name = peewee.CharField(max_length=320) email = peewee.CharField(max_length=320, index=True, unique=True) + password_hash = peewee.CharField(max_length=128, null=True) is_admin = peewee.BooleanField(default=False) class Meta: @@ -33,6 +36,12 @@ class User(BaseModel): def __unicode__(self): return '%r, %r' % (self.name, self.email) + def hash_password(self, password): + self.password_hash = pwd_context.encrypt(password) + + def verify_password(self, password): + return pwd_context.verify(password, self.password_hash) + class QueryResult(db.Model): id = peewee.PrimaryKeyField() diff --git a/requirements.txt b/requirements.txt index 478dbba78..09d9ffa7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ Flask==0.10.1 Flask-GoogleAuth==0.4 Flask-RESTful==0.2.10 +Flask-Login==0.2.9 +passlib==1.6.2 Jinja2==2.7.2 MarkupSafe==0.18 WTForms==1.0.5 diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 4d5a540f6..b904280b2 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -18,8 +18,7 @@ def authenticated_user(c, user=None): user = user_factory.create() with c.session_transaction() as sess: - sess['openid'] = {'email': user.email, 'name': user.name, - 'id': user.id, 'is_admin': user.is_admin} + sess['user_id'] = user.id yield @@ -58,33 +57,35 @@ class TestAuthentication(BaseTestCase): rv = c.get("/") self.assertEquals(302, rv.status_code) - def test_returns_content_when_authenticated_with_correct_domain(self): - settings.GOOGLE_APPS_DOMAIN = "example.com" - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@example.com")): - rv = c.get("/") - self.assertEquals(200, rv.status_code) + # This scenario can no longer be tested with authenticated_user decorator + # def test_returns_content_when_authenticated_with_correct_domain(self): + # settings.GOOGLE_APPS_DOMAIN = "example.com" + # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@example.com")): + # rv = c.get("/") + # self.assertEquals(200, rv.status_code) + # - def test_redirects_when_authenticated_with_wrong_domain(self): - settings.GOOGLE_APPS_DOMAIN = "example.com" - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): - rv = c.get("/") - self.assertEquals(302, rv.status_code) + # def test_redirects_when_authenticated_with_wrong_domain(self): + # settings.GOOGLE_APPS_DOMAIN = "example.com" + # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): + # rv = c.get("/") + # self.assertEquals(302, rv.status_code) - def test_returns_content_when_user_in_allowed_list(self): - settings.GOOGLE_APPS_DOMAIN = "example.com" - settings.ALLOWED_EXTERNAL_USERS = ["test@not-example.com"] - - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): - rv = c.get("/") - self.assertEquals(200, rv.status_code) - - def test_returns_content_when_google_apps_domain_empty(self): - settings.GOOGLE_APPS_DOMAIN = "" - settings.ALLOWED_EXTERNAL_USERS = [] - - with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@whatever.com")): - rv = c.get("/") - self.assertEquals(200, rv.status_code) + # def test_returns_content_when_user_in_allowed_list(self): + # settings.GOOGLE_APPS_DOMAIN = "example.com" + # settings.ALLOWED_EXTERNAL_USERS = ["test@not-example.com"] + # + # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): + # rv = c.get("/") + # self.assertEquals(200, rv.status_code) + + # def test_returns_content_when_google_apps_domain_empty(self): + # settings.GOOGLE_APPS_DOMAIN = "" + # settings.ALLOWED_EXTERNAL_USERS = [] + # + # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@whatever.com")): + # rv = c.get("/") + # self.assertEquals(200, rv.status_code) class PingTest(TestCase): From f334122e41061d49e71cb8f024926e3402910259 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 15:37:33 +0200 Subject: [PATCH 274/540] Add mock to dev_requirements --- dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev_requirements.txt b/dev_requirements.txt index 0b1ca3c15..d9c92539d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,2 +1,3 @@ nose==1.3.0 coverage==3.7.1 +mock==1.0.1 From 1277da7e92f664e15369ec4ae9f512fe034e1252 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 15:41:20 +0200 Subject: [PATCH 275/540] Chagne logging not to depend on app context --- redash/authentication.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index e40cf8729..5ddb2aa48 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -5,11 +5,12 @@ from flask import current_app, request, make_response, g, redirect, url_for from flask.ext.googleauth import GoogleAuth, login from flask.ext.login import LoginManager, login_user, current_user import time +import logging from werkzeug.contrib.fixers import ProxyFix from redash import models, settings login_manager = LoginManager() - +logger = logging.getLogger('authentication') def sign(key, path, expires): if not key: @@ -69,18 +70,18 @@ def validate_email(email): return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN) -def create_and_login_user(_, openid_user): +def create_and_login_user(app, openid_user): if not validate_email(openid_user.email): return try: user = models.User.get(models.User.email == openid_user.email) if user.name != openid_user.name: - current_app.logger.debug("Updating user name (%r -> %r)", user.name, openid_user.name) + logger.debug("Updating user name (%r -> %r)", user.name, openid_user.name) user.name = openid_user.name user.save() except models.User.DoesNotExist: - current_app.logger.debug("Creating user object (%r)", user.name) + logger.debug("Creating user object (%r)", openid_user.name) user = models.User.create(name=openid_user.name, email=openid_user.email) login_user(user, remember=True) From f557b53ce28874a908aac4b945fa7e48374f6102 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 15:41:38 +0200 Subject: [PATCH 276/540] Tests for authentication functions --- tests/test_authentication.py | 50 ++++++++++++++++++++++++++++++++++++ tests/test_controllers.py | 30 ---------------------- 2 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 tests/test_authentication.py diff --git a/tests/test_authentication.py b/tests/test_authentication.py new file mode 100644 index 000000000..6755d84f4 --- /dev/null +++ b/tests/test_authentication.py @@ -0,0 +1,50 @@ +from unittest import TestCase +from mock import patch +from flask_googleauth import ObjectDict +from tests import BaseTestCase +from redash.authentication import validate_email, create_and_login_user +from redash import settings, models +from tests.factories import user_factory + + +class TestEmailValidation(TestCase): + def test_accepts_address_with_correct_domain(self): + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'): + self.assertTrue(validate_email('example@example.com')) + + def test_accepts_address_from_exception_list(self): + with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ALLOWED_EXTERNAL_USERS=['whatever@whatever.com']): + self.assertTrue(validate_email('whatever@whatever.com')) + + def test_accept_any_address_when_domain_empty(self): + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', None): + self.assertTrue(validate_email('whatever@whatever.com')) + + def test_rejects_address_with_incorrect_domain(self): + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'): + self.assertFalse(validate_email('whatever@whatever.com')) + + +class TestCreateAndLoginUser(BaseTestCase): + def test_logins_valid_user(self): + user = user_factory.create(email='test@example.com') + + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'), patch('redash.authentication.login_user') as login_user_mock: + create_and_login_user(None, user) + login_user_mock.assert_called_once_with(user, remember=True) + + def test_creates_vaild_new_user(self): + user = ObjectDict({'email': 'test@example.com', 'name': 'Test User'}) + + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'), patch('redash.authentication.login_user') as login_user_mock: + create_and_login_user(None, user) + + self.assertEqual(models.User.select().count(), 1) + self.assertTrue(login_user_mock.called) + + def test_ignores_invliad_user(self): + user = ObjectDict({'email': 'test@whatever.com'}) + + with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'), patch('redash.authentication.login_user') as login_user_mock: + create_and_login_user(None, user) + self.assertFalse(login_user_mock.called) \ No newline at end of file diff --git a/tests/test_controllers.py b/tests/test_controllers.py index b904280b2..0e5d56e2a 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -57,36 +57,6 @@ class TestAuthentication(BaseTestCase): rv = c.get("/") self.assertEquals(302, rv.status_code) - # This scenario can no longer be tested with authenticated_user decorator - # def test_returns_content_when_authenticated_with_correct_domain(self): - # settings.GOOGLE_APPS_DOMAIN = "example.com" - # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@example.com")): - # rv = c.get("/") - # self.assertEquals(200, rv.status_code) - # - - # def test_redirects_when_authenticated_with_wrong_domain(self): - # settings.GOOGLE_APPS_DOMAIN = "example.com" - # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): - # rv = c.get("/") - # self.assertEquals(302, rv.status_code) - - # def test_returns_content_when_user_in_allowed_list(self): - # settings.GOOGLE_APPS_DOMAIN = "example.com" - # settings.ALLOWED_EXTERNAL_USERS = ["test@not-example.com"] - # - # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@not-example.com")): - # rv = c.get("/") - # self.assertEquals(200, rv.status_code) - - # def test_returns_content_when_google_apps_domain_empty(self): - # settings.GOOGLE_APPS_DOMAIN = "" - # settings.ALLOWED_EXTERNAL_USERS = [] - # - # with app.test_client() as c, authenticated_user(c, user=user_factory.create(email="test@whatever.com")): - # rv = c.get("/") - # self.assertEquals(200, rv.status_code) - class PingTest(TestCase): def test_ping(self): From 522536cfe0ff59418e7aa44c23627cf7ec992cb9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 15:46:29 +0200 Subject: [PATCH 277/540] CircleCi: install dev_requirements.txt --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 23c414e72..bd8edb32b 100644 --- a/circle.yml +++ b/circle.yml @@ -8,7 +8,7 @@ machine: dependencies: pre: - make deps - - pip install requests coverage nose + - pip install -r dev_requirements.txt - pip install -r requirements.txt cache_directories: - rd_ui/node_modules/ From 7271b7a5f0b9ed104a022d2972fefaa39afc3039 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 17:59:08 +0200 Subject: [PATCH 278/540] Login view --- rd_ui/app/login.html | 81 ++++++++++++++++++++++++++++++++++++++ rd_ui/app/styles/login.css | 37 +++++++++++++++++ redash/authentication.py | 7 +--- redash/controllers.py | 23 +++++++++-- redash/models.py | 2 +- redash/settings.py | 2 +- tests/test_controllers.py | 77 ++++++++++++++++++++++++++++++++++++ 7 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 rd_ui/app/login.html create mode 100644 rd_ui/app/styles/login.css diff --git a/rd_ui/app/login.html b/rd_ui/app/login.html new file mode 100644 index 000000000..3a96ffc07 --- /dev/null +++ b/rd_ui/app/login.html @@ -0,0 +1,81 @@ + + + + + + + re:dash Login + + + + + + + + + + + + + +
      +
      + +
      + +
      + + +
      +
      + + + +
      +
      + +
      + + + + + + +
      +
      + Google +
      +
      +
      + +
      +
      + + + + + + + \ No newline at end of file diff --git a/rd_ui/app/styles/login.css b/rd_ui/app/styles/login.css new file mode 100644 index 000000000..a418bbca5 --- /dev/null +++ b/rd_ui/app/styles/login.css @@ -0,0 +1,37 @@ +.main { + max-width: 320px; + margin: 0 auto; +} + +.login-or { + position: relative; + font-size: 18px; + color: #aaa; + margin-top: 10px; + margin-bottom: 10px; + padding-top: 10px; + padding-bottom: 10px; +} + +.span-or { + display: block; + position: absolute; + left: 50%; + top: -2px; + margin-left: -25px; + background-color: #fff; + width: 50px; + text-align: center; +} + +.hr-or { + background-color: #cdcdcd; + height: 1px; + margin-top: 0px !important; + margin-bottom: 0px !important; +} + +/*h3 {*/ + /*text-align: center;*/ + /*line-height: 300%;*/ +/*}*/ \ No newline at end of file diff --git a/redash/authentication.py b/redash/authentication.py index 5ddb2aa48..98ec9db29 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -55,10 +55,7 @@ class HMACAuthentication(object): if self.api_key_authentication(): return fn(*args, **kwargs) - blueprint = current_app.extensions['googleauth'].blueprint - # The make_response call is a work around for flask-restful testing only for - # flask.wrappers.Resource instead of werkzeug.wrappers.Response - return make_response(redirect(url_for("%s.login" % blueprint.name, next=request.url))) + return make_response(redirect(url_for("login", next=request.url))) return decorated @@ -95,7 +92,7 @@ def load_user(user_id): def setup_authentication(app): - openid_auth = GoogleAuth(app) + openid_auth = GoogleAuth(app, url_prefix="/google_auth") # If we don't have a list of external users, we can use Google's federated login, which limits # the domain with which you can sign in. if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN: diff --git a/redash/controllers.py b/redash/controllers.py index ff50d441b..fbaa79359 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -11,9 +11,9 @@ import numbers import cStringIO import datetime -from flask import g, render_template, send_from_directory, make_response, request, jsonify +from flask import render_template, send_from_directory, make_response, request, jsonify, redirect from flask.ext.restful import Resource, abort -from flask_login import current_user +from flask_login import current_user, login_user import sqlparse from redash import settings, utils @@ -49,6 +49,24 @@ def index(anything=None): return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) +@app.route('/login', methods=['GET', 'POST']) +def login(): + if current_user.is_authenticated(): + return redirect(request.args.get('next') or '/') + + if request.method == 'POST': + user = models.User.select().where(models.User.email == request.form['username']).first() + if user and user.verify_password(request.form['password']): + remember = ('remember' in request.form) + login_user(user, remember=remember) + return redirect(request.args.get('next') or '/') + + return render_template("login.html", + analytics=settings.ANALYTICS, + next=request.args.get('next'), + username=request.form.get('username', '')) + + @app.route('/status.json') @auth.required def status_api(): @@ -329,7 +347,6 @@ class JobAPI(BaseResource): api.add_resource(JobAPI, '/api/jobs/', endpoint='job') @app.route('/') -@auth.required def send_static(filename): return send_from_directory(settings.STATIC_ASSETS_PATH, filename) diff --git a/redash/models.py b/redash/models.py index a79bf442b..6d60254d3 100644 --- a/redash/models.py +++ b/redash/models.py @@ -40,7 +40,7 @@ class User(BaseModel, UserMixin): self.password_hash = pwd_context.encrypt(password) def verify_password(self, password): - return pwd_context.verify(password, self.password_hash) + return self.password_hash and pwd_context.verify(password, self.password_hash) class QueryResult(db.Model): diff --git a/redash/settings.py b/redash/settings.py index c20032851..90c5b45e1 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -51,7 +51,7 @@ GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") # Email addresses of admin users (comma separated) ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", '')) ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '')) -STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/")) +STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/")) WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 0e5d56e2a..109a9cf5e 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -2,6 +2,7 @@ from contextlib import contextmanager import json import time from unittest import TestCase +from mock import patch from tests import BaseTestCase from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ query_result_factory, user_factory @@ -333,3 +334,79 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)) self.assertEquals(rv.status_code, 200) + +class TestLogin(BaseTestCase): + def test_get_login_form(self): + with app.test_client() as c: + rv = c.get('/login') + self.assertEquals(rv.status_code, 200) + + def test_submit_non_existing_user(self): + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': 'arik', 'password': 'password'}) + self.assertEquals(rv.status_code, 200) + self.assertFalse(login_user_mock.called) + + def test_submit_correct_user_and_password(self): + user = user_factory.create() + user.hash_password('password') + user.save() + + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': user.email, 'password': 'password'}) + self.assertEquals(rv.status_code, 302) + login_user_mock.assert_called_with(user, remember=False) + + def test_submit_correct_user_and_password_and_remember_me(self): + user = user_factory.create() + user.hash_password('password') + user.save() + + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': user.email, 'password': 'password', 'remember': True}) + self.assertEquals(rv.status_code, 302) + login_user_mock.assert_called_with(user, remember=True) + + def test_submit_correct_user_and_password_with_next(self): + user = user_factory.create() + user.hash_password('password') + user.save() + + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login?next=/test', + data={'username': user.email, 'password': 'password'}) + self.assertEquals(rv.status_code, 302) + self.assertEquals(rv.location, 'http://localhost/test') + login_user_mock.assert_called_with(user, remember=False) + + def test_submit_incorrect_user(self): + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': 'non-existing', 'password': 'password'}) + self.assertEquals(rv.status_code, 200) + self.assertFalse(login_user_mock.called) + + def test_submit_incorrect_password(self): + user = user_factory.create() + user.hash_password('password') + user.save() + + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': user.email, 'password': 'badbadpassword'}) + self.assertEquals(rv.status_code, 200) + self.assertFalse(login_user_mock.called) + + def test_submit_incorrect_password(self): + user = user_factory.create() + + with app.test_client() as c, patch('redash.controllers.login_user') as login_user_mock: + rv = c.post('/login', data={'username': user.email, 'password': ''}) + self.assertEquals(rv.status_code, 200) + self.assertFalse(login_user_mock.called) + + def test_user_already_loggedin(self): + with app.test_client() as c, authenticated_user(c), patch('redash.controllers.login_user') as login_user_mock: + rv = c.get('/login') + self.assertEquals(rv.status_code, 302) + self.assertFalse(login_user_mock.called) + + # brute force protection? \ No newline at end of file From d348fe9012766681abc165924cf8c702bd41259f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 18:27:05 +0200 Subject: [PATCH 279/540] Logout controller --- redash/controllers.py | 12 ++++++++++-- tests/test_controllers.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index fbaa79359..e4a2c1b55 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -11,9 +11,10 @@ import numbers import cStringIO import datetime -from flask import render_template, send_from_directory, make_response, request, jsonify, redirect +from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \ + session from flask.ext.restful import Resource, abort -from flask_login import current_user, login_user +from flask_login import current_user, login_user, logout_user import sqlparse from redash import settings, utils @@ -67,6 +68,13 @@ def login(): username=request.form.get('username', '')) +@app.route('/logout') +def logout(): + logout_user() + session.pop('openid', None) + + return redirect('/login') + @app.route('/status.json') @auth.required def status_api(): diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 109a9cf5e..19d9cdb4f 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -2,6 +2,7 @@ from contextlib import contextmanager import json import time from unittest import TestCase +from flask.ext.login import current_user from mock import patch from tests import BaseTestCase from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ @@ -409,4 +410,20 @@ class TestLogin(BaseTestCase): self.assertEquals(rv.status_code, 302) self.assertFalse(login_user_mock.called) - # brute force protection? \ No newline at end of file + # TODO: brute force protection? + + +class TestLogout(BaseTestCase): + def test_logout_when_not_loggedin(self): + with app.test_client() as c: + rv = c.get('/logout') + self.assertEquals(rv.status_code, 302) + self.assertFalse(current_user.is_authenticated()) + + def test_logout_when_loggedin(self): + with app.test_client() as c, authenticated_user(c): + rv = c.get('/') + self.assertTrue(current_user.is_authenticated()) + rv = c.get('/logout') + self.assertEquals(rv.status_code, 302) + self.assertFalse(current_user.is_authenticated()) \ No newline at end of file From e82f561c0327c67820847e854a56a570993339e8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 18:30:06 +0200 Subject: [PATCH 280/540] BaseResource.current_user wrapper to get real user object. --- redash/controllers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index e4a2c1b55..b93835a8e 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -113,6 +113,10 @@ class BaseResource(Resource): super(BaseResource, self).__init__(*args, **kwargs) self._user = None + @property + def current_user(self): + return current_user._get_current_object() + class DashboardListAPI(BaseResource): def get(self): @@ -124,7 +128,7 @@ class DashboardListAPI(BaseResource): def post(self): dashboard_properties = request.get_json(force=True) dashboard = models.Dashboard(name=dashboard_properties['name'], - user=current_user.id, + user=self.current_user, layout='[]') dashboard.save() return dashboard.to_dict() @@ -212,7 +216,7 @@ class QueryListAPI(BaseResource): for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: query_def.pop(field, None) - query_def['user'] = current_user.id + query_def['user'] = self.current_user query = models.Query(**query_def) query.save() From aa2bf4fe22b8597f90f2e5b7b2b94556a40a9c62 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 2 Mar 2014 21:54:50 +0200 Subject: [PATCH 281/540] Ability to disable openid or password login --- rd_ui/app/login.html | 4 ++++ redash/authentication.py | 16 +++++++--------- redash/controllers.py | 9 +++++++-- redash/settings.py | 8 ++++++++ tests/test_controllers.py | 11 +++++++++++ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/login.html b/rd_ui/app/login.html index 3a96ffc07..a5271f9d4 100644 --- a/rd_ui/app/login.html +++ b/rd_ui/app/login.html @@ -56,6 +56,8 @@ + {% if show_google_openid %} +
      + + {% endif %}
      diff --git a/redash/authentication.py b/redash/authentication.py index 98ec9db29..238683d7a 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -23,9 +23,6 @@ def sign(key, path, expires): class HMACAuthentication(object): - def __init__(self, auth): - self.auth = auth - @staticmethod def api_key_authentication(): signature = request.args.get('signature') @@ -92,14 +89,15 @@ def load_user(user_id): def setup_authentication(app): - openid_auth = GoogleAuth(app, url_prefix="/google_auth") - # If we don't have a list of external users, we can use Google's federated login, which limits - # the domain with which you can sign in. - if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN: - openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN + if settings.GOOGLE_OPENID_ENABLED: + openid_auth = GoogleAuth(app, url_prefix="/google_auth") + # If we don't have a list of external users, we can use Google's federated login, which limits + # the domain with which you can sign in. + if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN: + openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN login_manager.init_app(app) app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = settings.COOKIE_SECRET - return HMACAuthentication(openid_auth) + return HMACAuthentication() diff --git a/redash/controllers.py b/redash/controllers.py index b93835a8e..74ceebb40 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -12,7 +12,7 @@ import cStringIO import datetime from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \ - session + session, url_for from flask.ext.restful import Resource, abort from flask_login import current_user, login_user, logout_user @@ -55,6 +55,10 @@ def login(): if current_user.is_authenticated(): return redirect(request.args.get('next') or '/') + if not settings.PASSWORD_LOGIN_ENABLED: + blueprint = app.extensions['googleauth'].blueprint + return redirect(url_for("%s.login" % blueprint.name, next=request.args.get('next'))) + if request.method == 'POST': user = models.User.select().where(models.User.email == request.form['username']).first() if user and user.verify_password(request.form['password']): @@ -65,7 +69,8 @@ def login(): return render_template("login.html", analytics=settings.ANALYTICS, next=request.args.get('next'), - username=request.form.get('username', '')) + username=request.form.get('username', ''), + show_google_openid=settings.GOOGLE_OPENID_ENABLED) @app.route('/logout') diff --git a/redash/settings.py b/redash/settings.py index 90c5b45e1..adb1b882a 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -1,3 +1,4 @@ +import json import os import urlparse @@ -32,6 +33,11 @@ def array_from_string(str): return array + +def parse_boolean(str): + return json.loads(str.lower()) + + REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379") # "pg", "graphite" or "mysql" @@ -48,6 +54,8 @@ DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed # access GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "") +GOOGLE_OPENID_ENABLED = parse_boolean(os.environ.get("REDASH_GOOGLE_OPENID_ENABLED", "true")) +PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "false")) # Email addresses of admin users (comma separated) ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", '')) ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '')) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 19d9cdb4f..d370c2d6f 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -2,6 +2,7 @@ from contextlib import contextmanager import json import time from unittest import TestCase +from flask import url_for from flask.ext.login import current_user from mock import patch from tests import BaseTestCase @@ -337,6 +338,16 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): class TestLogin(BaseTestCase): + def setUp(self): + settings.PASSWORD_LOGIN_ENABLED = True + super(TestLogin, self).setUp() + + def test_redirects_to_google_login_if_password_disabled(self): + with app.test_client() as c, patch.object(settings, 'PASSWORD_LOGIN_ENABLED', False): + rv = c.get('/login') + self.assertEquals(rv.status_code, 302) + self.assertTrue(rv.location.endswith(url_for('GoogleAuth.login'))) + def test_get_login_form(self): with app.test_client() as c: rv = c.get('/login') From 0c9fa8b51b7363cb684525fafdbbec3cb98414d2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 11:49:31 +0200 Subject: [PATCH 282/540] Build assets for login page --- rd_ui/Gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/Gruntfile.js b/rd_ui/Gruntfile.js index 374e0cd6e..8ed52f3b8 100644 --- a/rd_ui/Gruntfile.js +++ b/rd_ui/Gruntfile.js @@ -170,7 +170,7 @@ module.exports = function (grunt) { } }, useminPrepare: { - html: '<%= yeoman.app %>/index.html', + html: ['<%= yeoman.app %>/index.html', '<%= yeoman.app %>/login.html'], options: { dest: '<%= yeoman.dist %>' } From 62108e3dac796398f2d82194982863f65c5762d7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 11:53:49 +0200 Subject: [PATCH 283/540] Set is_admin of user based on ADMINS list. --- redash/authentication.py | 3 ++- tests/test_authentication.py | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index 238683d7a..8676fa1e7 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -76,7 +76,8 @@ def create_and_login_user(app, openid_user): user.save() except models.User.DoesNotExist: logger.debug("Creating user object (%r)", openid_user.name) - user = models.User.create(name=openid_user.name, email=openid_user.email) + user = models.User.create(name=openid_user.name, email=openid_user.email, + is_admin=(openid_user.email in settings.ADMINS)) login_user(user, remember=True) diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 6755d84f4..3fbe8a413 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -34,13 +34,29 @@ class TestCreateAndLoginUser(BaseTestCase): login_user_mock.assert_called_once_with(user, remember=True) def test_creates_vaild_new_user(self): - user = ObjectDict({'email': 'test@example.com', 'name': 'Test User'}) + openid_user = ObjectDict({'email': 'test@example.com', 'name': 'Test User'}) - with patch.object(settings, 'GOOGLE_APPS_DOMAIN', 'example.com'), patch('redash.authentication.login_user') as login_user_mock: - create_and_login_user(None, user) + with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \ + patch('redash.authentication.login_user') as login_user_mock: + + create_and_login_user(None, openid_user) - self.assertEqual(models.User.select().count(), 1) self.assertTrue(login_user_mock.called) + user = models.User.get(models.User.email == openid_user.email) + + self.assertFalse(user.is_admin) + + def test_creates_vaild_new_user_and_sets_is_admin(self): + openid_user = ObjectDict({'email': 'admin@example.com', 'name': 'Test User'}) + + with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \ + patch('redash.authentication.login_user') as login_user_mock: + + create_and_login_user(None, openid_user) + + self.assertTrue(login_user_mock.called) + user = models.User.get(models.User.email == openid_user.email) + self.assertTrue(user.is_admin) def test_ignores_invliad_user(self): user = ObjectDict({'email': 'test@whatever.com'}) From b8a0077b1d7e60fc3d0a48c6f71420cbd9ec14ba Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 12:18:15 +0200 Subject: [PATCH 284/540] user management commands --- manage.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 9627fd7a9..cfae89920 100755 --- a/manage.py +++ b/manage.py @@ -10,10 +10,11 @@ atfork.stdlib_fixer.fix_logging_module() import logging import time from redash import settings, app, db, models, data_manager, __version__ -from flask.ext.script import Manager +from flask.ext.script import Manager, prompt_pass manager = Manager(app) database_manager = Manager(help="Manages the database (create/drop tables).") +users_manager = Manager(help="Users management commands.") @manager.command def version(): @@ -63,7 +64,34 @@ def drop_tables(): create_db(False, True) + +@users_manager.option('email', help="User's email") +@users_manager.option('name', help="User's full name") +@users_manager.option('--admin', dest='is_admin', default=False, help="set user as admin") +@users_manager.option('--google', dest='google_auth', default=False, help="user uses Google Auth to login") +def create(email, name, is_admin=False, google_auth=False): + print "Creating user (%s, %s)..." % (email, name) + print "Admin: %r" % is_admin + print "Login with Google Auth: %r\n" % google_auth + + user = models.User(email=email, name=name, is_admin=is_admin) + if not google_auth: + password = prompt_pass("Password") + user.hash_password(password) + + try: + user.save() + except Exception, e: + print "Failed creating user: %s" % e.message + + +@users_manager.option('email', help="email address of user to delete") +def delete(email): + deleted_count = models.User.delete().where(models.User.email == email).execute() + print "Deleted %d users." % deleted_count + manager.add_command("database", database_manager) +manager.add_command("users", users_manager) if __name__ == '__main__': channel = logging.StreamHandler() From 81122c9865dc9c62165eb21bed023d4f1fcebb20 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 13:07:57 +0200 Subject: [PATCH 285/540] Fix: create_user_and_login should accept user --- redash/authentication.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/redash/authentication.py b/redash/authentication.py index 8676fa1e7..52b6a3de6 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -64,22 +64,22 @@ def validate_email(email): return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN) -def create_and_login_user(app, openid_user): - if not validate_email(openid_user.email): +def create_and_login_user(app, user): + if not validate_email(user.email): return try: - user = models.User.get(models.User.email == openid_user.email) - if user.name != openid_user.name: - logger.debug("Updating user name (%r -> %r)", user.name, openid_user.name) - user.name = openid_user.name - user.save() + user_object = models.User.get(models.User.email == user.email) + if user_object.name != user.name: + logger.debug("Updating user name (%r -> %r)", user_object.name, user.name) + user_object.name = user.name + user_object.save() except models.User.DoesNotExist: - logger.debug("Creating user object (%r)", openid_user.name) - user = models.User.create(name=openid_user.name, email=openid_user.email, - is_admin=(openid_user.email in settings.ADMINS)) + logger.debug("Creating user object (%r)", user.name) + user_object = models.User.create(name=user.name, email=user.email, + is_admin=(user.email in settings.ADMINS)) - login_user(user, remember=True) + login_user(user_object, remember=True) login.connect(create_and_login_user) From 3680d0c65d56bc4fcbeaf38b14328a53ec9ecd2d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 15:21:49 +0200 Subject: [PATCH 286/540] Fix: graphs with category as x axis were shown as datetime graphs, because drawChart is called twice and on second pass there is no x attribute on point object. --- rd_ui/app/scripts/ng-highchart.js | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index a5eb378d5..59ddc7b2a 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -210,29 +210,32 @@ scope.chart.series[0].remove(false); }; + if (_.some(scope.series[0].data, function (p) { - return angular.isString(p.x) + return (angular.isString(p.x) || angular.isDefined(p.name)); })) { scope.chart.xAxis[0].update({type: 'category'}); - // We need to make sure that for each category, each series has a value. - var categories = _.union.apply(this, _.map(scope.series, function (s) { - return _.pluck(s.data, 'x') - })); + if (!angular.isDefined(scope.series[0].data[0].name)) { + // We need to make sure that for each category, each series has a value. + var categories = _.union.apply(this, _.map(scope.series, function (s) { + return _.pluck(s.data, 'x') + })); - _.each(scope.series, function (s) { - // TODO: move this logic to Query#getChartData - var yValues = _.groupBy(s.data, 'x'); + _.each(scope.series, function (s) { + // TODO: move this logic to Query#getChartData + var yValues = _.groupBy(s.data, 'x'); - var newData = _.sortBy(_.map(categories, function (category) { - return { - name: category, - y: yValues[category] && yValues[category][0].y - } - }), 'name'); + var newData = _.sortBy(_.map(categories, function (category) { + return { + name: category, + y: yValues[category] && yValues[category][0].y + } + }), 'name'); - s.data = newData; - }); + s.data = newData; + }); + } } else { scope.chart.xAxis[0].update({type: 'datetime'}); } From ac946fd014da865d2b015ba2df291ff33666d627 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 15:27:39 +0200 Subject: [PATCH 287/540] Feature: sort category charts by y value. --- rd_ui/app/scripts/ng-highchart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 59ddc7b2a..c68d3d109 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -231,7 +231,7 @@ name: category, y: yValues[category] && yValues[category][0].y } - }), 'name'); + }), 'y').reverse(); s.data = newData; }); From 8ccbe9c069c45ef89555d9ee8d9c48f8c752611a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 18:26:15 +0200 Subject: [PATCH 288/540] Update the refresh queries query --- redash/data/manager.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index 5061dff0f..c6d28b3b2 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -99,21 +99,20 @@ class Manager(object): return job def refresh_queries(self): - sql = """SELECT queries.query, queries.ttl, retrieved_at - FROM (SELECT query, min(ttl) as ttl FROM queries WHERE ttl > 0 GROUP by query) queries - JOIN (SELECT query, max(retrieved_at) as retrieved_at - FROM query_results - GROUP BY query) query_results on query_results.query=queries.query - WHERE queries.ttl > 0 - AND query_results.retrieved_at + ttl * interval '1 second' < now() at time zone 'utc';""" + sql = """SELECT first_value(t1."query") over(partition by t1.query_hash) + FROM "queries" AS t1 + INNER JOIN "query_results" AS t2 ON (t1."latest_query_data_id" = t2."id") + WHERE ((t1."ttl" > 0) AND ((t2."retrieved_at" + t1.ttl * interval '1 second') < + now() at time zone 'utc')); + """ self.status['last_refresh_at'] = time.time() self._save_status() logging.info("Refreshing queries...") queries = self.run_query(sql) - for query, ttl, retrieved_at in queries: - self.add_job(query, worker.Job.LOW_PRIORITY) + for query in queries: + self.add_job(query[0], worker.Job.LOW_PRIORITY) logging.info("Done refreshing queries... %d" % len(queries)) From 2bcb56d2490d25f79cfe30f55c13d540d2f87911 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Mon, 3 Mar 2014 16:41:53 +0000 Subject: [PATCH 289/540] Fixing PR #112 as discussed with Arik --- rd_ui/app/scripts/controllers.js | 7 +++++++ rd_ui/app/scripts/directives.js | 24 +----------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index 46eaa2e8c..ac9905161 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -282,6 +282,13 @@ }); } }; + + unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { + if (newPanel && $routeParams.queryId == undefined) { + unbind(); + $scope.saveQuery(); + } + }); } var QueriesCtrl = function($scope, $http, $location, $filter, Query) { diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 17156b2ef..da8b387e9 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -46,7 +46,7 @@ } }]); - directives.directive('editVisulatizationForm', ['Visualization', 'growl', '$location', function(Visualization, growl, $location) { + directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { return { restrict: 'E', templateUrl: '/views/edit_visualization.html', @@ -142,28 +142,6 @@ }; scope.submit = function() { - if (!scope.query.id) { - scope.$parent.saveQuery(false, null, false); - var unwatch = scope.$watch('query', function(q) { - if (q && q.id) { - unwatch(); - scope.vis['query_id'] = q.id; - - scope.saveVisualization(false); - - $location.path($location.path().replace('new', q.id)).replace(); - } - }, true); - } else { - scope.saveVisualization(); - } - }; - - scope.saveVisualization = function(updateTabs) { - if (updateTabs == undefined) { - updateTabs = true; - } - Visualization.save(scope.vis, function success(result) { growl.addSuccessMessage("Visualization saved"); From ffb2ec9bd16349a4fce53307f049ba35ed28eb10 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Mon, 3 Mar 2014 16:45:45 +0000 Subject: [PATCH 290/540] Fixing PR #112 as discussed with Arik --- rd_ui/app/scripts/controllers.js | 8 ++------ rd_ui/app/scripts/directives.js | 24 +++++++++--------------- rd_ui/app/scripts/services.js | 2 +- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/rd_ui/app/scripts/controllers.js b/rd_ui/app/scripts/controllers.js index ac9905161..0fcc4d058 100644 --- a/rd_ui/app/scripts/controllers.js +++ b/rd_ui/app/scripts/controllers.js @@ -130,15 +130,11 @@ }) } - $scope.saveQuery = function (duplicate, oldId, updateUrl) { + $scope.saveQuery = function (duplicate, oldId) { if (!oldId) { oldId = $scope.query.id; } - if (updateUrl == undefined) { - updateUrl = true; - } - delete $scope.query.latest_query_data; $scope.query.$save(function (q) { pristineHash = q.getHash(); @@ -151,7 +147,7 @@ } if (oldId != q.id) { - if (oldId == undefined && updateUrl) { + if (oldId == undefined) { $location.path($location.path().replace('new', q.id)).replace(); } else { // TODO: replace this with a safer method diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index da8b387e9..762cdd88f 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -76,7 +76,7 @@ }; scope.stacking = "none"; - + if (!scope.vis) { // create new visualization // wait for query to load to populate with defaults @@ -92,7 +92,7 @@ } }, true); } - + function newOptions(chartType) { if (chartType === Visualization.prototype.TYPES.CHART) { return { @@ -145,19 +145,13 @@ Visualization.save(scope.vis, function success(result) { growl.addSuccessMessage("Visualization saved"); - if (updateTabs) { - scope.vis = { - 'query_id': scope.query.id - }; - - var visIds = _.pluck(scope.query.visualizations, 'id'); - var index = visIds.indexOf(result.id); - - if (index > -1) { - scope.query.visualizations[index] = result; - } else { - scope.query.visualizations.push(result); - } + var visIds = _.pluck(scope.query.visualizations, 'id'); + var index = visIds.indexOf(result.id); + + if (index > -1) { + scope.query.visualizations[index] = result; + } else { + scope.query.visualizations.push(result); } }, function error() { growl.addErrorMessage("Visualization could not be saved"); diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 3693ca707..7216c2d2b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -279,7 +279,7 @@ Query.prototype.getHash = function() { return [this.name, this.description, this.query].join('!#'); }; - + return Query; }; From 0822789002958b01e243c5df7bf44413dab3eed8 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Mon, 3 Mar 2014 17:08:07 +0000 Subject: [PATCH 291/540] Fixing PR #112 as discussed with Arik --- rd_ui/app/scripts/directives.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 762cdd88f..ec42ac6ab 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -145,6 +145,8 @@ Visualization.save(scope.vis, function success(result) { growl.addSuccessMessage("Visualization saved"); + scope.vis = result; + var visIds = _.pluck(scope.query.visualizations, 'id'); var index = visIds.indexOf(result.id); From 55cc3dd90ec9ec74d79ebe7b01ecd2860b88d5d1 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Mon, 3 Mar 2014 17:11:38 +0000 Subject: [PATCH 292/540] Fixing PR #112 as discussed with Arik --- rd_ui/app/scripts/directives.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index ec42ac6ab..9b7412505 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -83,7 +83,7 @@ var unwatch = scope.$watch('query', function(q) { if (q && q.id) { unwatch(); - + if (!scope.vis) { scope.vis = { 'query_id': q.id, @@ -144,12 +144,12 @@ scope.submit = function() { Visualization.save(scope.vis, function success(result) { growl.addSuccessMessage("Visualization saved"); - + scope.vis = result; - + var visIds = _.pluck(scope.query.visualizations, 'id'); var index = visIds.indexOf(result.id); - + if (index > -1) { scope.query.visualizations[index] = result; } else { From 53ef0f3f2dcc1b158bbd304a42b790e11375d931 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:15:02 +0200 Subject: [PATCH 293/540] Add statsd client requirements. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 09d9ffa7f..c5a2bb1cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ wsgiref==0.1.2 wtf-peewee==0.2.2 Flask-Script==0.6.6 honcho==0.5.0 +statsd==2.1.2 From 3f302ee4a32e4c18bb52e1192b0af5796c2f5e89 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:15:14 +0200 Subject: [PATCH 294/540] Statsd settings. --- redash/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/redash/settings.py b/redash/settings.py index adb1b882a..b06bf4976 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -40,6 +40,10 @@ def parse_boolean(str): REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379") +STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1") +STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125")) +STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash") + # "pg", "graphite" or "mysql" CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # Connection string for the database that is used to run queries against. Examples: From c7af5bdce94bcea3877e031fa2ff242ef2df0fd0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:16:42 +0200 Subject: [PATCH 295/540] Use integers instead of uuid for workers id. --- redash/data/manager.py | 4 ++-- redash/data/worker.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index c6d28b3b2..80ce8996a 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -168,8 +168,8 @@ class Manager(object): runner = query_runner.redshift(connection_string) redis_connection_params = self.redis_connection.connection_pool.connection_kwargs - self.workers = [worker.Worker(self, redis_connection_params, runner) - for _ in range(workers_count)] + self.workers = [worker.Worker(worker_id, self, redis_connection_params, runner) + for worker_id in range(workers_count)] for w in self.workers: w.start() diff --git a/redash/data/worker.py b/redash/data/worker.py index 1913baf88..c00178e87 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -144,7 +144,7 @@ class Job(object): class Worker(threading.Thread): - def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1): + def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1): self.manager = manager self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() @@ -153,8 +153,9 @@ class Worker(threading.Thread): self.query_runner = query_runner self.sleep_time = sleep_time self.child_pid = None - self.worker_id = uuid.uuid1() + self.worker_id = worker_id self.status = { + 'id': self.worker_id, 'jobs_count': 0, 'cancelled_jobs_count': 0, 'done_jobs_count': 0, From e43366f4221840a8787da1d89e7718cb4a8f0c49 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:17:25 +0200 Subject: [PATCH 296/540] Basic stats reporting. --- manage.py | 1 + redash/__init__.py | 5 ++++- redash/data/manager.py | 22 ++++++++++++++++++++-- redash/data/worker.py | 9 +++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/manage.py b/manage.py index cfae89920..52f02c46d 100755 --- a/manage.py +++ b/manage.py @@ -38,6 +38,7 @@ def runworkers(): while True: try: data_manager.refresh_queries() + data_manager.report_status() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) diff --git a/redash/__init__.py b/redash/__init__.py index a949a2d8b..05e94a270 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -5,6 +5,7 @@ from flask.ext.restful import Api from flask_peewee.db import Database import redis +from statsd import StatsClient from redash import settings, utils __version__ = '0.3.3' @@ -36,9 +37,11 @@ if redis_url.path: redis_db = redis_url.path[1] else: redis_db = 0 + redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password) +statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) from redash import data -data_manager = data.Manager(redis_connection, db) +data_manager = data.Manager(redis_connection, db, statsd_client) from redash import controllers diff --git a/redash/data/manager.py b/redash/data/manager.py index 80ce8996a..ca6c1c529 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -9,7 +9,6 @@ import logging import psycopg2 import qr import redis -from redash import settings from redash.data import worker from redash.utils import gen_query_hash @@ -24,7 +23,8 @@ class QueryResult(collections.namedtuple('QueryData', 'id query data runtime ret class Manager(object): - def __init__(self, redis_connection, db): + def __init__(self, redis_connection, db, statsd_client): + self.statsd_client = statsd_client self.redis_connection = redis_connection self.db = db self.workers = [] @@ -98,6 +98,20 @@ class Manager(object): return job + def report_status(self): + workers = [self.redis_connection.hgetall(w) + for w in self.redis_connection.smembers('workers')] + + for w in workers: + self.statsd_client.gauge('worker_{}.seconds_since_update'.format(w['id']), + time.time() - float(w['updated_at'])) + self.statsd_client.gauge('worker_{}.jobs_received'.format(w['id']), int(w['jobs_count'])) + self.statsd_client.gauge('worker_{}.jobs_done'.format(w['id']), int(w['done_jobs_count'])) + + manager_status = self.redis_connection.hgetall('manager:status') + self.statsd_client.gauge('manager.seconds_since_refresh', + time.time() - float(manager_status['last_refresh_at'])) + def refresh_queries(self): sql = """SELECT first_value(t1."query") over(partition by t1.query_hash) FROM "queries" AS t1 @@ -111,9 +125,13 @@ class Manager(object): logging.info("Refreshing queries...") queries = self.run_query(sql) + for query in queries: self.add_job(query[0], worker.Job.LOW_PRIORITY) + self.statsd_client.gauge('manager.outdated_queries', len(queries)) + self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs')) + logging.info("Done refreshing queries... %d" % len(queries)) def store_query_result(self, query, data, run_time, retrieved_at): diff --git a/redash/data/worker.py b/redash/data/worker.py index c00178e87..00b0f01ae 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -11,8 +11,9 @@ import time import signal import setproctitle import redis +from statsd import StatsClient from redash.utils import gen_query_hash - +from redash import settings class Job(object): HIGH_PRIORITY = 1 @@ -147,6 +148,8 @@ class Worker(threading.Thread): def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1): self.manager = manager + self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, + prefix=settings.STATSD_PREFIX) self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() if k in ('host', 'db', 'password', 'port')} self.continue_working = True @@ -242,7 +245,9 @@ class Worker(threading.Thread): annotated_query = job.query # TODO: here's the part that needs to be forked, not all of the worker process... - data, error = self.query_runner(annotated_query) + with self.statsd_client.timer('worker_{}.query_runner.run_time'.format(self.worker_id)): + data, error = self.query_runner(annotated_query) + run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", self.name, job.id, data and len(data), error) From b5d2285b998e51827d2e996939a2f4474ea6ec10 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:27:04 +0200 Subject: [PATCH 297/540] merge conflict fix & bump version --- rd_ui/app/scripts/ng-highchart.js | 2 +- redash/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 6a7c31acc..861047966 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -211,7 +211,7 @@ }; if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) { - return angular.isString(p.x) + return angular.isString(p.x) || angular.isDefined(p.name)); })) { scope.chart.xAxis[0].update({type: 'category'}); diff --git a/redash/__init__.py b/redash/__init__.py index 05e94a270..0f2af7d57 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -8,7 +8,7 @@ import redis from statsd import StatsClient from redash import settings, utils -__version__ = '0.3.3' +__version__ = '0.3.5' app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, From a91eb9435b4ea094739e19cb5c97b348e7ac84b8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:29:43 +0200 Subject: [PATCH 298/540] typo fix --- rd_ui/app/scripts/ng-highchart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 861047966..6e4f25b59 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -211,7 +211,7 @@ }; if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) { - return angular.isString(p.x) || angular.isDefined(p.name)); + return (angular.isString(p.x) || angular.isDefined(p.name)); })) { scope.chart.xAxis[0].update({type: 'category'}); From 14ecfd2cc8006358657a57930083b9a7ceb5e184 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 3 Mar 2014 20:40:50 +0200 Subject: [PATCH 299/540] Fix: set the chart name by default to chart type. --- rd_ui/app/scripts/directives.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 9b7412505..6f1719891 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -87,6 +87,7 @@ if (!scope.vis) { scope.vis = { 'query_id': q.id, + 'type': Visualization.prototype.TYPES.CHART }; } } @@ -109,6 +110,12 @@ var chartOptionsUnwatch = null; scope.$watch('vis.type', function(type) { + // if not edited by user, set name to match type + if (type && scope.vis && !scope.visForm.name.$dirty) { + // poor man's titlecase + scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase(); + } + if (type && type == Visualization.prototype.TYPES.CHART) { if (scope.vis.options.series.stacking === null) { scope.stacking = "none"; From 2c904641a519759d87818aaf4a78bdf45060faa0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 14 Feb 2014 13:24:14 +0200 Subject: [PATCH 300/540] Remove unused dependency. --- rd_ui/app/scripts/directives.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 6f1719891..a19216c77 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -3,7 +3,7 @@ var directives = angular.module('redash.directives', []); - directives.directive('rdTab', ['$location', function($location) { + directives.directive('rdTab', function() { return { restrict: 'E', scope: { @@ -19,9 +19,9 @@ }); } } - }]); + }); - directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) { + directives.directive('rdTabs', ['$location', function($location) { return { restrict: 'E', scope: { From 79e40a667b67800846fd0661ca6a86e0981d43e4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 15 Feb 2014 12:21:13 +0200 Subject: [PATCH 301/540] Refactor visualizations: The main code doesn't know about individual visualizations and each visualization is contained in its own module. Should make adding/editing/removing visualizations easier. --- rd_ui/app/index.html | 4 + rd_ui/app/scripts/app.js | 4 + rd_ui/app/scripts/directives.js | 124 ------------ rd_ui/app/scripts/query_fiddle/renderers.js | 179 +----------------- rd_ui/app/scripts/services.js | 16 -- rd_ui/app/scripts/visualizations/base.js | 144 ++++++++++++++ rd_ui/app/scripts/visualizations/chart.js | 100 ++++++++++ rd_ui/app/scripts/visualizations/cohort.js | 56 ++++++ rd_ui/app/scripts/visualizations/table.js | 87 +++++++++ rd_ui/app/views/edit_visualization.html | 24 --- rd_ui/app/views/queryfiddle.html | 4 +- .../views/visualizations/chart_editor.html | 7 + .../visualizations/edit_visualization.html | 18 ++ 13 files changed, 423 insertions(+), 344 deletions(-) create mode 100644 rd_ui/app/scripts/visualizations/base.js create mode 100644 rd_ui/app/scripts/visualizations/chart.js create mode 100644 rd_ui/app/scripts/visualizations/cohort.js create mode 100644 rd_ui/app/scripts/visualizations/table.js delete mode 100644 rd_ui/app/views/edit_visualization.html create mode 100644 rd_ui/app/views/visualizations/chart_editor.html create mode 100644 rd_ui/app/views/visualizations/edit_visualization.html diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 004da8099..05145c8c5 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -111,6 +111,10 @@ + + + + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 04718e989..09329ee36 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -5,6 +5,10 @@ angular.module('redash', [ 'redash.filters', 'redash.services', 'redash.renderers', + 'redash.visualization', + 'redash.visualization.chart', + 'redash.visualization.cohort', + 'redash.visualization.table', 'ui.codemirror', 'highchart', 'angular-growl', diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a19216c77..83be258e4 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -46,130 +46,6 @@ } }]); - directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) { - return { - restrict: 'E', - templateUrl: '/views/edit_visualization.html', - replace: true, - scope: { - query: '=', - vis: '=?' - }, - link: function(scope, element, attrs) { - scope.advancedMode = false; - scope.visTypes = { - 'Chart': Visualization.prototype.TYPES.CHART, - 'Cohort': Visualization.prototype.TYPES.COHORT - }; - scope.seriesTypes = { - 'Line': 'line', - 'Column': 'column', - 'Area': 'area', - 'Scatter': 'scatter', - 'Pie': 'pie' - }; - - scope.stackingOptions = { - "None": "none", - "Normal": "normal", - "Percent": "percent" - }; - - scope.stacking = "none"; - - if (!scope.vis) { - // create new visualization - // wait for query to load to populate with defaults - var unwatch = scope.$watch('query', function(q) { - if (q && q.id) { - unwatch(); - - if (!scope.vis) { - scope.vis = { - 'query_id': q.id, - 'type': Visualization.prototype.TYPES.CHART - }; - } - } - }, true); - } - - function newOptions(chartType) { - if (chartType === Visualization.prototype.TYPES.CHART) { - return { - 'series': { - 'type': 'column', - 'stacking': null - } - }; - }; - - return {}; - } - - var chartOptionsUnwatch = null; - - scope.$watch('vis.type', function(type) { - // if not edited by user, set name to match type - if (type && scope.vis && !scope.visForm.name.$dirty) { - // poor man's titlecase - scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase(); - } - - if (type && type == Visualization.prototype.TYPES.CHART) { - if (scope.vis.options.series.stacking === null) { - scope.stacking = "none"; - } else if (scope.vis.options.series.stacking === undefined) { - scope.stacking = "normal"; - } else { - scope.stacking = scope.vis.options.series.stacking ; - } - - chartOptionsUnwatch = scope.$watch("stacking", function(stacking) { - if (stacking == "none") { - scope.vis.options.series.stacking = null; - } else { - scope.vis.options.series.stacking = stacking; - } - }); - } else { - if (chartOptionsUnwatch) { - chartOptionsUnwatch(); - chartOptionsUnwatch = null; - } - } - }); - - scope.toggleAdvancedMode = function() { - scope.advancedMode = !scope.advancedMode; - }; - - scope.typeChanged = function() { - scope.vis.options = newOptions(scope.vis.type); - }; - - scope.submit = function() { - Visualization.save(scope.vis, function success(result) { - growl.addSuccessMessage("Visualization saved"); - - scope.vis = result; - - var visIds = _.pluck(scope.query.visualizations, 'id'); - var index = visIds.indexOf(result.id); - - if (index > -1) { - scope.query.visualizations[index] = result; - } else { - scope.query.visualizations.push(result); - } - }, function error() { - growl.addErrorMessage("Visualization could not be saved"); - }); - }; - } - } - }]); - directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) { return { restrict: 'E', diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index e747ac182..1eb965e16 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -1,134 +1,5 @@ var renderers = angular.module('redash.renderers', []); -renderers.directive('visualizationRenderer', function() { - return { - restrict: 'E', - scope: { - visualization: '=', - queryResult: '=' - }, - template: '
      ' + - '' + - '' + - '' + - '
      ', - replace: false - } -}); - -renderers.directive('chartRenderer', function () { - return { - restrict: 'E', - scope: { - queryResult: '=', - options: '=?' - }, - template: "", - replace: false, - controller: ['$scope', function ($scope) { - $scope.chartSeries = []; - $scope.chartOptions = {}; - - $scope.$watch('options', function(chartOptions) { - if (chartOptions) { - $scope.chartOptions = chartOptions; - } - }); - $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data || $scope.queryResult.getData() == null) { - $scope.chartSeries.splice(0, $scope.chartSeries.length); - } else { - $scope.chartSeries.splice(0, $scope.chartSeries.length); - - _.each($scope.queryResult.getChartData(), function (s) { - $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'})); - }); - } - }); - }] - } -}) - -renderers.directive('gridRenderer', function () { - return { - restrict: 'E', - scope: { - queryResult: '=', - itemsPerPage: '=' - }, - templateUrl: "/views/grid_renderer.html", - replace: false, - controller: ['$scope', function ($scope) { - $scope.gridColumns = []; - $scope.gridData = []; - $scope.gridConfig = { - isPaginationEnabled: true, - itemsByPage: $scope.itemsPerPage || 15, - maxSize: 8 - }; - - $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data) { - return; - } - - if ($scope.queryResult.getData() == null) { - $scope.gridColumns = []; - $scope.gridData = []; - $scope.filters = []; - } else { - - - $scope.filters = $scope.queryResult.getFilters(); - - var gridData = _.map($scope.queryResult.getData(), function (row) { - var newRow = {}; - _.each(row, function (val, key) { - newRow[$scope.queryResult.getColumnCleanName(key)] = val; - }) - return newRow; - }); - - $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { - var columnDefinition = { - 'label': $scope.queryResult.getColumnFriendlyNames()[i], - 'map': col - }; - - if (gridData.length > 0) { - var exampleData = gridData[0][col]; - if (angular.isNumber(exampleData)) { - columnDefinition['formatFunction'] = 'number'; - columnDefinition['formatParameter'] = 2; - } else if (moment.isMoment(exampleData)) { - columnDefinition['formatFunction'] = function(value) { - return value.format("DD/MM/YY HH:mm"); - } - } - } - - return columnDefinition; - }); - - $scope.gridData = _.clone(gridData); - - $scope.$watch('filters', function (filters) { - $scope.gridData = _.filter(gridData, function (row) { - return _.reduce(filters, function (memo, filter) { - if (filter.current == 'All') { - return memo && true; - } - - return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current); - }, true); - }); - }, true); - } - }); - }] - } -}) - renderers.directive('pivotTableRenderer', function () { return { restrict: 'E', @@ -152,52 +23,4 @@ renderers.directive('pivotTableRenderer', function () { }); } } -}) - -renderers.directive('cohortRenderer', function() { - return { - restrict: 'E', - scope: { - queryResult: '=' - }, - template: "", - replace: false, - link: function($scope, element, attrs) { - $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data) { - return; - } - - if ($scope.queryResult.getData() == null) { - - } else { - var sortedData = _.sortBy($scope.queryResult.getData(), "date"); - var grouped = _.groupBy(sortedData, "date"); - var data = _.map(grouped, function(values, date) { - var row = [values[0].total]; - _.each(values, function(value) { row.push(value.value); }); - return row; - }); - - var initialDate = moment(sortedData[0].date).toDate(), - container = angular.element(element)[0]; - - Cornelius.draw({ - initialDate: initialDate, - container: container, - cohort: data, - title: null, - timeInterval: 'daily', - labels: { - time: 'Activation Day', - people: 'Users' - }, - formatHeaderLabel: function (i) { - return "Day " + (i - 1); - } - }); - } - }); - } - } -}) +}); \ No newline at end of file diff --git a/rd_ui/app/scripts/services.js b/rd_ui/app/scripts/services.js index 7216c2d2b..cf7efc58b 100644 --- a/rd_ui/app/scripts/services.js +++ b/rd_ui/app/scripts/services.js @@ -283,23 +283,7 @@ return Query; }; - var Visualization = function($resource) { - var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); - - Visualization.prototype = { - TYPES: { - 'CHART': 'CHART', - 'COHORT': 'COHORT', - 'TABLE': 'TABLE' - } - }; - - return Visualization; - }; - angular.module('redash.services', []) .factory('QueryResult', ['$resource', '$timeout', QueryResult]) .factory('Query', ['$resource', 'QueryResult', Query]) - .factory('Visualization', ['$resource', Visualization]) - })(); diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js new file mode 100644 index 000000000..538f0c872 --- /dev/null +++ b/rd_ui/app/scripts/visualizations/base.js @@ -0,0 +1,144 @@ +(function () { + var Visualization = function() { + this.visualizations = {}; + this.visualizationTypes = {}; + + this.registerVisualization = function(type, name, rendererTemplate, editorTemplate, defaultOptions, skipTypes) { + var visualization = { + rendererTemplate: rendererTemplate, + editorTemplate: editorTemplate, + type: type, + name: name, + defaultOptions: defaultOptions || {} + }; + + // TODO: this is prone to errors; better refactor. + if (_.isEmpty(this.visualizations)) { + this.defaultVisualization = visualization; + } + + this.visualizations[type] = visualization; + + if (!skipTypes) { + this.visualizationTypes[name] = type; + }; + }; + + this.getSwitchTemplate = function(property) { + var pattern = /(<[a-zA-Z0-9-]*? )/ + + var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) { + if (visualization[property]) { + var ngSwitch = '$1ng-switch-when="' + visualization.type + '" '; + var template = visualization[property].replace(pattern, ngSwitch); + + return templates + "\n" + template; + } + + return templates; + }, ""); + + mergedTemplates = '
      '+ mergedTemplates + "
      "; + + return mergedTemplates; + } + + this.$get = ['$resource', function($resource) { + var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); + Visualization.visualization = this.visualizations; + Visualization.visualizationTypes = this.visualizationTypes; + Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('rendererTemplate'); + Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); + Visualization.defaultVisualization = this.defaultVisualization; + + return Visualization; + }]; + }; + + var VisualizationRenderer = function(Visualization) { + return { + restrict: 'E', + scope: { + visualization: '=', + queryResult: '=' + }, + // TODO: using switch here (and in the options editor) might introduce errors and bad + // performance wise. It's better to eventually show the correct template based on the + // visualization type and not make the browser render all of them. + template: Visualization.renderVisualizationsTemplate, + replace: false + } + }; + + var VisualizationOptionsEditor = function(Visualization) { + return { + restrict: 'E', + template: Visualization.editorTemplate, + replace: false + } + }; + + var EditVisualizationForm = function(Visualization, growl) { + return { + restrict: 'E', + templateUrl: '/views/visualizations/edit_visualization.html', + replace: true, + scope: { + query: '=', + visualization: '=?' + }, + link: function (scope, element, attrs) { + scope.visTypes = Visualization.visualizationTypes; + + if (!scope.visualization) { + // create new visualization + // wait for query to load to populate with defaults + var unwatch = scope.$watch('query', function (q) { + if (q && q.id) { + unwatch(); + scope.visualization = { + 'query_id': q.id, + 'type': Visualization.defaultVisualization.type, + 'name': '', + 'description': q.description || '', + 'options': Visualization.defaultVisualization.defaultOptions + }; + } + }, true); + } + + scope.$watch('visualization.type', function (type) { + // if not edited by user, set name to match type + if (type && scope.visualization && !scope.visForm.name.$dirty) { + // poor man's titlecase + scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase(); + } + }); + + scope.submit = function () { + Visualization.save(scope.visualization, function success(result) { + growl.addSuccessMessage("Visualization saved"); + + scope.visualization = result; + + var visIds = _.pluck(scope.query.visualizations, 'id'); + var index = visIds.indexOf(result.id); + if (index > -1) { + scope.query.visualizations[index] = result; + } else { + scope.query.visualizations.push(result); + } + }, function error() { + growl.addErrorMessage("Visualization could not be saved"); + }); + }; + } + } + }; + + angular.module('redash.visualization', []) + .provider('Visualization', Visualization) + .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) + .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor]) + .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm]) +})(); diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js new file mode 100644 index 000000000..3618a64c3 --- /dev/null +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -0,0 +1,100 @@ +(function () { + var chartVisualization = angular.module('redash.visualization.chart', []); + + chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) { + var renderTemplate = ''; + var editTemplate = ''; + var defaultOptions = { + 'series': { + 'type': 'column', + 'stacking': null + } + }; + VisualizationProvider.registerVisualization('CHART', 'Chart', renderTemplate, editTemplate, defaultOptions); + }]); + + chartVisualization.directive('chartRenderer', function () { + return { + restrict: 'E', + scope: { + queryResult: '=', + options: '=?' + }, + template: "", + replace: false, + controller: ['$scope', function ($scope) { + $scope.chartSeries = []; + $scope.chartOptions = {}; + + $scope.$watch('options', function(chartOptions) { + if (chartOptions) { + $scope.chartOptions = chartOptions; + } + }); + $scope.$watch('queryResult && queryResult.getData()', function (data) { + if (!data || $scope.queryResult.getData() == null) { + $scope.chartSeries.splice(0, $scope.chartSeries.length); + } else { + $scope.chartSeries.splice(0, $scope.chartSeries.length); + + _.each($scope.queryResult.getChartData(), function (s) { + $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'})); + }); + } + }); + }] + } + }); + + chartVisualization.directive('chartEditor', function () { + return { + restrict: 'E', + templateUrl: '/views/visualizations/chart_editor.html', + replace: true, + link: function (scope, element, attrs) { + scope.seriesTypes = { + 'Line': 'line', + 'Column': 'column', + 'Area': 'area', + 'Scatter': 'scatter', + 'Pie': 'pie' + }; + + scope.stackingOptions = { + "None": "none", + "Normal": "normal", + "Percent": "percent" + }; + + scope.stacking = "none"; + + var chartOptionsUnwatch = null; + + scope.$watch('visualization', function (visualization) { + if (visualization && visualization.type == 'CHART') { + if (scope.visualization.options.series.stacking === null) { + scope.stacking = "none"; + } else if (scope.visualization.options.series.stacking === undefined) { + scope.stacking = "normal"; + } else { + scope.stacking = scope.visualization.options.series.stacking; + } + + chartOptionsUnwatch = scope.$watch("stacking", function (stacking) { + if (stacking == "none") { + scope.visualization.options.series.stacking = null; + } else { + scope.visualization.options.series.stacking = stacking; + } + }); + } else { + if (chartOptionsUnwatch) { + chartOptionsUnwatch(); + chartOptionsUnwatch = null; + } + } + }); + } + } + }); +}()); \ No newline at end of file diff --git a/rd_ui/app/scripts/visualizations/cohort.js b/rd_ui/app/scripts/visualizations/cohort.js new file mode 100644 index 000000000..fd72896f8 --- /dev/null +++ b/rd_ui/app/scripts/visualizations/cohort.js @@ -0,0 +1,56 @@ +(function () { + var cohortVisualization = angular.module('redash.visualization.cohort', []); + + cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) { + VisualizationProvider.registerVisualization('COHORT', 'Cohort', '', null); + }]); + + cohortVisualization.directive('cohortRenderer', function() { + return { + restrict: 'E', + scope: { + queryResult: '=' + }, + template: "", + replace: false, + link: function($scope, element, attrs) { + $scope.$watch('queryResult && queryResult.getData()', function (data) { + if (!data) { + return; + } + + if ($scope.queryResult.getData() == null) { + + } else { + var sortedData = _.sortBy($scope.queryResult.getData(), "date"); + var grouped = _.groupBy(sortedData, "date"); + var data = _.map(grouped, function(values, date) { + var row = [values[0].total]; + _.each(values, function(value) { row.push(value.value); }); + return row; + }); + + var initialDate = moment(sortedData[0].date).toDate(), + container = angular.element(element)[0]; + + Cornelius.draw({ + initialDate: initialDate, + container: container, + cohort: data, + title: null, + timeInterval: 'daily', + labels: { + time: 'Activation Day', + people: 'Users' + }, + formatHeaderLabel: function (i) { + return "Day " + (i - 1); + } + }); + } + }); + } + } + }); + +}()); \ No newline at end of file diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js new file mode 100644 index 000000000..cf63fd530 --- /dev/null +++ b/rd_ui/app/scripts/visualizations/table.js @@ -0,0 +1,87 @@ +(function () { + var tableVisualization = angular.module('redash.visualization.table', []); + + tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) { + VisualizationProvider.registerVisualization('TABLE', 'Table', '', null, {}, true); + }]); + + tableVisualization.directive('gridRenderer', function () { + return { + restrict: 'E', + scope: { + queryResult: '=', + itemsPerPage: '=' + }, + templateUrl: "/views/grid_renderer.html", + replace: false, + controller: ['$scope', function ($scope) { + $scope.gridColumns = []; + $scope.gridData = []; + $scope.gridConfig = { + isPaginationEnabled: true, + itemsByPage: $scope.itemsPerPage || 15, + maxSize: 8 + }; + + $scope.$watch('queryResult && queryResult.getData()', function (data) { + if (!data) { + return; + } + + if ($scope.queryResult.getData() == null) { + $scope.gridColumns = []; + $scope.gridData = []; + $scope.filters = []; + } else { + + + $scope.filters = $scope.queryResult.getFilters(); + + var gridData = _.map($scope.queryResult.getData(), function (row) { + var newRow = {}; + _.each(row, function (val, key) { + newRow[$scope.queryResult.getColumnCleanName(key)] = val; + }) + return newRow; + }); + + $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { + var columnDefinition = { + 'label': $scope.queryResult.getColumnFriendlyNames()[i], + 'map': col + }; + + if (gridData.length > 0) { + var exampleData = gridData[0][col]; + if (angular.isNumber(exampleData)) { + columnDefinition['formatFunction'] = 'number'; + columnDefinition['formatParameter'] = 2; + } else if (moment.isMoment(exampleData)) { + columnDefinition['formatFunction'] = function(value) { + return value.format("DD/MM/YY HH:mm"); + } + } + } + + return columnDefinition; + }); + + $scope.gridData = _.clone(gridData); + + $scope.$watch('filters', function (filters) { + $scope.gridData = _.filter(gridData, function (row) { + return _.reduce(filters, function (memo, filter) { + if (filter.current == 'All') { + return memo && true; + } + + return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current); + }, true); + }); + }, true); + } + }); + }] + } + }) +}()); \ No newline at end of file diff --git a/rd_ui/app/views/edit_visualization.html b/rd_ui/app/views/edit_visualization.html deleted file mode 100644 index 2521902f3..000000000 --- a/rd_ui/app/views/edit_visualization.html +++ /dev/null @@ -1,24 +0,0 @@ -
      -
      - - -
      - -
      - - -
      - -
      - - - - - -
      - -
      - -
      - -
      diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 3a51ea073..3c74fefdb 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -73,7 +73,7 @@

      - +

      @@ -90,7 +90,7 @@

      - +
      diff --git a/rd_ui/app/views/visualizations/chart_editor.html b/rd_ui/app/views/visualizations/chart_editor.html new file mode 100644 index 000000000..2c36235a4 --- /dev/null +++ b/rd_ui/app/views/visualizations/chart_editor.html @@ -0,0 +1,7 @@ +
      + + + + + +
      \ No newline at end of file diff --git a/rd_ui/app/views/visualizations/edit_visualization.html b/rd_ui/app/views/visualizations/edit_visualization.html new file mode 100644 index 000000000..ca504a11a --- /dev/null +++ b/rd_ui/app/views/visualizations/edit_visualization.html @@ -0,0 +1,18 @@ +
      +
      + + +
      + +
      + + +
      + + + +
      + +
      + +
      From 4630a8d18d6e5b2361a045301075bad563cc2b7f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 15 Feb 2014 19:29:15 +0200 Subject: [PATCH 302/540] Fix switch regular expression. --- rd_ui/app/scripts/visualizations/base.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 538f0c872..0c3e85154 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -25,11 +25,11 @@ }; this.getSwitchTemplate = function(property) { - var pattern = /(<[a-zA-Z0-9-]*? )/ + var pattern = /(<[a-zA-Z0-9-]*?)( |>)/ var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) { if (visualization[property]) { - var ngSwitch = '$1ng-switch-when="' + visualization.type + '" '; + var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2'; var template = visualization[property].replace(pattern, ngSwitch); return templates + "\n" + template; @@ -85,6 +85,7 @@ replace: true, scope: { query: '=', + queryResult: '=', visualization: '=?' }, link: function (scope, element, attrs) { From 69f5de64785a554a8a401e48b7bedf0b3272abda Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 15 Feb 2014 19:30:19 +0200 Subject: [PATCH 303/540] Fix chart editor declaration. --- rd_ui/app/scripts/visualizations/chart.js | 3 +-- rd_ui/app/views/queryfiddle.html | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js index 3618a64c3..78c73dc78 100644 --- a/rd_ui/app/scripts/visualizations/chart.js +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -3,7 +3,7 @@ chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) { var renderTemplate = ''; - var editTemplate = ''; + var editTemplate = ''; var defaultOptions = { 'series': { 'type': 'column', @@ -50,7 +50,6 @@ return { restrict: 'E', templateUrl: '/views/visualizations/chart_editor.html', - replace: true, link: function (scope, element, attrs) { scope.seriesTypes = { 'Line': 'line', diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html index 3c74fefdb..99b3cf3e4 100644 --- a/rd_ui/app/views/queryfiddle.html +++ b/rd_ui/app/views/queryfiddle.html @@ -73,7 +73,7 @@

      - +

      From 6d93ccc0d054a3b6867fb1a9b900d1f37cb28864 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 16 Feb 2014 10:40:57 +0200 Subject: [PATCH 304/540] Remove the need to declare each visualization module in app dependencies. --- rd_ui/app/scripts/app.js | 3 --- rd_ui/app/scripts/visualizations/chart.js | 2 +- rd_ui/app/scripts/visualizations/cohort.js | 2 +- rd_ui/app/scripts/visualizations/table.js | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 09329ee36..52ac7029d 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -6,9 +6,6 @@ angular.module('redash', [ 'redash.services', 'redash.renderers', 'redash.visualization', - 'redash.visualization.chart', - 'redash.visualization.cohort', - 'redash.visualization.table', 'ui.codemirror', 'highchart', 'angular-growl', diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js index 78c73dc78..7be1d9855 100644 --- a/rd_ui/app/scripts/visualizations/chart.js +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -1,5 +1,5 @@ (function () { - var chartVisualization = angular.module('redash.visualization.chart', []); + var chartVisualization = angular.module('redash.visualization'); chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) { var renderTemplate = ''; diff --git a/rd_ui/app/scripts/visualizations/cohort.js b/rd_ui/app/scripts/visualizations/cohort.js index fd72896f8..07c6d24a1 100644 --- a/rd_ui/app/scripts/visualizations/cohort.js +++ b/rd_ui/app/scripts/visualizations/cohort.js @@ -1,5 +1,5 @@ (function () { - var cohortVisualization = angular.module('redash.visualization.cohort', []); + var cohortVisualization = angular.module('redash.visualization'); cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) { VisualizationProvider.registerVisualization('COHORT', 'Cohort', '', null); diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index cf63fd530..1513ce1eb 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -1,5 +1,5 @@ (function () { - var tableVisualization = angular.module('redash.visualization.table', []); + var tableVisualization = angular.module('redash.visualization'); tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) { VisualizationProvider.registerVisualization('TABLE', 'Table', '', null, {}, true); From a5971b0c6920176beabe7cc787bd5532840ec7a9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 10:14:36 +0200 Subject: [PATCH 305/540] typo fix --- rd_ui/app/scripts/visualizations/base.js | 3 ++- rd_ui/app/views/visualizations/edit_visualization.html | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 0c3e85154..37c51b634 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -45,7 +45,7 @@ this.$get = ['$resource', function($resource) { var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); - Visualization.visualization = this.visualizations; + Visualization.visualizations = this.visualizations; Visualization.visualizationTypes = this.visualizationTypes; Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('rendererTemplate'); Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); @@ -97,6 +97,7 @@ var unwatch = scope.$watch('query', function (q) { if (q && q.id) { unwatch(); + scope.visualization = { 'query_id': q.id, 'type': Visualization.defaultVisualization.type, diff --git a/rd_ui/app/views/visualizations/edit_visualization.html b/rd_ui/app/views/visualizations/edit_visualization.html index ca504a11a..0250bc4bb 100644 --- a/rd_ui/app/views/visualizations/edit_visualization.html +++ b/rd_ui/app/views/visualizations/edit_visualization.html @@ -1,7 +1,7 @@
      - +
      From bfeded207a2d9d8f1e4498abd02ce30df4ae2175 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 10:19:28 +0200 Subject: [PATCH 306/540] Rename Visaulization to VisualizationProvider --- rd_ui/app/scripts/visualizations/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 37c51b634..5e98bff04 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -1,5 +1,5 @@ (function () { - var Visualization = function() { + var VisualizationProvider = function() { this.visualizations = {}; this.visualizationTypes = {}; @@ -139,7 +139,7 @@ }; angular.module('redash.visualization', []) - .provider('Visualization', Visualization) + .provider('Visualization', VisualizationProvider) .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor]) .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm]) From 68482afa5cac08ad226b4efdd32af3ce913ede79 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 10:59:07 +0200 Subject: [PATCH 307/540] Fix: reset visualization form after saving visualization --- rd_ui/app/scripts/visualizations/base.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 5e98bff04..6d63adfec 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -91,6 +91,16 @@ link: function (scope, element, attrs) { scope.visTypes = Visualization.visualizationTypes; + scope.newVisualization = function(q) { + return { + 'query_id': q.id, + 'type': Visualization.defaultVisualization.type, + 'name': '', + 'description': q.description || '', + 'options': Visualization.defaultVisualization.defaultOptions + }; + } + if (!scope.visualization) { // create new visualization // wait for query to load to populate with defaults @@ -98,13 +108,7 @@ if (q && q.id) { unwatch(); - scope.visualization = { - 'query_id': q.id, - 'type': Visualization.defaultVisualization.type, - 'name': '', - 'description': q.description || '', - 'options': Visualization.defaultVisualization.defaultOptions - }; + scope.visualization = scope.newVisualization(q); } }, true); } @@ -121,7 +125,7 @@ Visualization.save(scope.visualization, function success(result) { growl.addSuccessMessage("Visualization saved"); - scope.visualization = result; + scope.visualization = scope.newVisualization(scope.query); var visIds = _.pluck(scope.query.visualizations, 'id'); var index = visIds.indexOf(result.id); From 3320de07f2e87c8bd1a9cc76f26e15f4e35d9a6b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 11:09:04 +0200 Subject: [PATCH 308/540] Switch to config object instead of millions of params --- rd_ui/app/scripts/visualizations/base.js | 23 +++++++++++----------- rd_ui/app/scripts/visualizations/chart.js | 9 ++++++++- rd_ui/app/scripts/visualizations/cohort.js | 6 +++++- rd_ui/app/scripts/visualizations/table.js | 7 ++++++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 6d63adfec..340b469bf 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -2,25 +2,24 @@ var VisualizationProvider = function() { this.visualizations = {}; this.visualizationTypes = {}; + var defaultConfig = { + defaultOptions: {}, + skipTypes: false, + editorTemplate: null + } - this.registerVisualization = function(type, name, rendererTemplate, editorTemplate, defaultOptions, skipTypes) { - var visualization = { - rendererTemplate: rendererTemplate, - editorTemplate: editorTemplate, - type: type, - name: name, - defaultOptions: defaultOptions || {} - }; + this.registerVisualization = function(config) { + var visualization = _.extend({}, defaultConfig, config); // TODO: this is prone to errors; better refactor. if (_.isEmpty(this.visualizations)) { this.defaultVisualization = visualization; } - this.visualizations[type] = visualization; + this.visualizations[config.type] = visualization; - if (!skipTypes) { - this.visualizationTypes[name] = type; + if (!config.skipTypes) { + this.visualizationTypes[config.name] = config.type; }; }; @@ -47,7 +46,7 @@ var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); Visualization.visualizations = this.visualizations; Visualization.visualizationTypes = this.visualizationTypes; - Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('rendererTemplate'); + Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate'); Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); Visualization.defaultVisualization = this.defaultVisualization; diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js index 7be1d9855..60ef89408 100644 --- a/rd_ui/app/scripts/visualizations/chart.js +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -10,7 +10,14 @@ 'stacking': null } }; - VisualizationProvider.registerVisualization('CHART', 'Chart', renderTemplate, editTemplate, defaultOptions); + + VisualizationProvider.registerVisualization({ + type: 'CHART', + name: 'Chart', + renderTemplate: renderTemplate, + editorTemplate: editTemplate, + defaultOptions: defaultOptions + }); }]); chartVisualization.directive('chartRenderer', function () { diff --git a/rd_ui/app/scripts/visualizations/cohort.js b/rd_ui/app/scripts/visualizations/cohort.js index 07c6d24a1..4226c4a38 100644 --- a/rd_ui/app/scripts/visualizations/cohort.js +++ b/rd_ui/app/scripts/visualizations/cohort.js @@ -2,7 +2,11 @@ var cohortVisualization = angular.module('redash.visualization'); cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) { - VisualizationProvider.registerVisualization('COHORT', 'Cohort', '', null); + VisualizationProvider.registerVisualization({ + type: 'COHORT', + name: 'Cohort', + renderTemplate: '' + }); }]); cohortVisualization.directive('cohortRenderer', function() { diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index 1513ce1eb..98ee06d99 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -2,7 +2,12 @@ var tableVisualization = angular.module('redash.visualization'); tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) { - VisualizationProvider.registerVisualization('TABLE', 'Table', '', null, {}, true); + VisualizationProvider.registerVisualization({ + type: 'TABLE', + name: 'Table', + renderTemplate: '', + skipTypes: true + }); }]); tableVisualization.directive('gridRenderer', function () { From 9d4655cc0045de6c0d1e4837840c63462f84ef30 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 11:33:11 +0200 Subject: [PATCH 309/540] Fixes #81: reset query when saving widget --- rd_ui/app/scripts/directives.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index 83be258e4..d9d3fa473 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -157,6 +157,7 @@ $scope.widgetSize = 1; $scope.queryId = null; $scope.selectedVis = null; + $scope.query = null; } From dd5fd72bd266d4814961d3d4f84ae8ea429f7816 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 4 Mar 2014 11:38:00 +0200 Subject: [PATCH 310/540] Set default name when creating --- rd_ui/app/scripts/visualizations/base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 340b469bf..7d9e5071b 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -94,7 +94,7 @@ return { 'query_id': q.id, 'type': Visualization.defaultVisualization.type, - 'name': '', + 'name': Visualization.defaultVisualization.name, 'description': q.description || '', 'options': Visualization.defaultVisualization.defaultOptions }; From 54453ee9e5656173bbfb14782cd13b131432b57b Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Tue, 4 Mar 2014 13:04:55 +0000 Subject: [PATCH 311/540] Adding a new table called activity_log to log who runs what and when --- migrations/create_activity_log.py | 15 +++++++++++++++ redash/controllers.py | 6 ++++++ redash/models.py | 25 ++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 migrations/create_activity_log.py diff --git a/migrations/create_activity_log.py b/migrations/create_activity_log.py new file mode 100644 index 000000000..c0d8e23f1 --- /dev/null +++ b/migrations/create_activity_log.py @@ -0,0 +1,15 @@ +import json +import itertools +import peewee +from playhouse.migrate import Migrator +from redash import db, settings +from redash import models + +if __name__ == '__main__': + db.connect_db() + + if not models.ActivityLog.table_exists(): + print "Creating activity_log table..." + models.ActivityLog.create_table() + + db.close_db(None) \ No newline at end of file diff --git a/redash/controllers.py b/redash/controllers.py index 74ceebb40..6fb35055e 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -296,6 +296,12 @@ api.add_resource(VisualizationAPI, '/api/visualizations/', end class QueryResultListAPI(BaseResource): def post(self): params = request.json + + models.ActivityLog( + user=self.current_user, + type=1, + activity=params['query'] + ).save() if params['ttl'] == 0: query_result = None diff --git a/redash/models.py b/redash/models.py index 6d60254d3..69fa1d304 100644 --- a/redash/models.py +++ b/redash/models.py @@ -43,6 +43,29 @@ class User(BaseModel, UserMixin): return self.password_hash and pwd_context.verify(password, self.password_hash) +class ActivityLog(BaseModel): + id = peewee.PrimaryKeyField() + user = peewee.ForeignKeyField(User) + type = peewee.IntegerField() # 1 for query execution + activity = peewee.TextField() + created_at = peewee.DateTimeField(default=datetime.datetime.now) + + class Meta: + db_table = 'activity_log' + + def to_dict(self): + return { + 'id': self.id, + 'user': self.user.to_dict(), + 'type': self.type, + 'activity': self.activity, + 'created_at': self.created_at + } + + def __unicode__(self): + return unicode(self.id) + + class QueryResult(db.Model): id = peewee.PrimaryKeyField() query_hash = peewee.CharField(max_length=32, index=True) @@ -274,7 +297,7 @@ class Widget(db.Model): def __unicode__(self): return u"%s" % self.id -all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget) +all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog) def create_db(create_tables, drop_tables): From 4fef4a8d3394684afd2b1b91b23c18b57afe15d4 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Tue, 4 Mar 2014 13:12:43 +0000 Subject: [PATCH 312/540] Removing unneeded imports in migration script --- migrations/create_activity_log.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/migrations/create_activity_log.py b/migrations/create_activity_log.py index c0d8e23f1..b2d612c25 100644 --- a/migrations/create_activity_log.py +++ b/migrations/create_activity_log.py @@ -1,8 +1,4 @@ -import json -import itertools -import peewee -from playhouse.migrate import Migrator -from redash import db, settings +from redash import db from redash import models if __name__ == '__main__': From 100fd2c9f0baf92c81377585c7fd1f6184d264e6 Mon Sep 17 00:00:00 2001 From: Eran Kampf Date: Tue, 4 Mar 2014 15:21:03 +0200 Subject: [PATCH 313/540] Handle empty data returned by queries --- redash/data/query_runner_mysql.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/redash/data/query_runner_mysql.py b/redash/data/query_runner_mysql.py index a479322e9..bedb11ed7 100644 --- a/redash/data/query_runner_mysql.py +++ b/redash/data/query_runner_mysql.py @@ -28,18 +28,24 @@ def mysql(connection_string): data = cursor.fetchall() - num_fields = len(cursor.description) - column_names = [i[0] for i in cursor.description] + cursor_desc = cursor.description + if (cursor_desc != None): + num_fields = len(cursor_desc) + column_names = [i[0] for i in cursor.description] - rows = [dict(zip(column_names, row)) for row in data] + rows = [dict(zip(column_names, row)) for row in data] - columns = [{'name': col_name, - 'friendly_name': col_name, - 'type': None} for col_name in column_names] + columns = [{'name': col_name, + 'friendly_name': col_name, + 'type': None} for col_name in column_names] - data = {'columns': columns, 'rows': rows} - json_data = json.dumps(data, cls=JSONEncoder) - error = None + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + else: + json_data = None + error = "No data was returned." + cursor.close() except MySQLdb.Error, e: json_data = None From e270d2534fcb55e19be2bbd872354b9554bda2a8 Mon Sep 17 00:00:00 2001 From: Christopher Valles Date: Tue, 4 Mar 2014 14:36:57 +0000 Subject: [PATCH 314/540] Adding a constant to activity model --- redash/controllers.py | 2 +- redash/models.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/redash/controllers.py b/redash/controllers.py index 6fb35055e..7f8ce455b 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -299,7 +299,7 @@ class QueryResultListAPI(BaseResource): models.ActivityLog( user=self.current_user, - type=1, + type=models.ActivityLog.QUERY_EXECUTION, activity=params['query'] ).save() diff --git a/redash/models.py b/redash/models.py index 69fa1d304..d398589ff 100644 --- a/redash/models.py +++ b/redash/models.py @@ -44,6 +44,8 @@ class User(BaseModel, UserMixin): class ActivityLog(BaseModel): + QUERY_EXECUTION = 1 + id = peewee.PrimaryKeyField() user = peewee.ForeignKeyField(User) type = peewee.IntegerField() # 1 for query execution From 7c89ff5c1b31024d8f41d8517dae61169588e0a1 Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Wed, 5 Mar 2014 08:46:09 +0200 Subject: [PATCH 315/540] No need to use github style authenticated URLs, it can just break things. --- rd_ui/bower.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 321f39913..179b3176b 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -16,8 +16,8 @@ "angular-resource": "1.0.7", "angular-growl": "0.3.1", "angular-route": "1.2.7", - "pivottable": "git@github.com:arikfr/pivottable.git#master", - "cornelius": "git@github.com:restorando/cornelius.git", + "pivottable": "https://github.com/arikfr/pivottable.git", + "cornelius": "https://github.com/restorando/cornelius.git", "gridster": "0.2.0", "mousetrap": "~1.4.6" }, @@ -26,7 +26,6 @@ "angular-scenario": "~1.0.7" }, "resolutions": { - "angular": "~1.2.7", - "jquery": "~1.9.1" + "angular": "~1.2.7" } } From 5b42a4b36e6ce45afed0dafd415cc8cf2d5e7e0a Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Wed, 5 Mar 2014 08:46:27 +0200 Subject: [PATCH 316/540] Bigquery support --- redash/data/manager.py | 6 +- redash/data/query_runner_bigquery.py | 94 ++++++++++++++++++++++++++++ redash/settings.py | 2 + 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 redash/data/query_runner_bigquery.py diff --git a/redash/data/manager.py b/redash/data/manager.py index ca6c1c529..55a10066d 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -169,7 +169,7 @@ class Manager(object): def start_workers(self, workers_count, connection_type, connection_string): if self.workers: return self.workers - + if connection_type == 'mysql': from redash.data import query_runner_mysql runner = query_runner_mysql.mysql(connection_string) @@ -181,6 +181,10 @@ class Manager(object): else: connection_params['auth'] = None runner = query_runner_graphite.graphite(connection_params) + elif connection_type == 'bigquery': + from redash.data import query_runner_bigquery + connection_params = json.loads(connection_string) + runner = query_runner_bigquery.bigquery(connection_params) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_bigquery.py b/redash/data/query_runner_bigquery.py new file mode 100644 index 000000000..d6929273b --- /dev/null +++ b/redash/data/query_runner_bigquery.py @@ -0,0 +1,94 @@ +import httplib2 +import json +import logging +import sys + +import apiclient.errors +from apiclient.discovery import build +from apiclient.errors import HttpError +from oauth2client.client import SignedJwtAssertionCredentials + +from redash.utils import JSONEncoder + +def bigquery(connection_string): + def load_key(filename): + f = file(filename, "rb") + try: + return f.read() + finally: + f.close() + + def get_bigquery_service(): + scope = [ + "https://www.googleapis.com/auth/bigquery", + ] + + credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"], load_key(connection_string["privateKey"]), scope=scope) + http = httplib2.Http() + http = credentials.authorize(http) + + return build("bigquery", "v2", http=http) + + def query_runner(query): + bigquery_service = get_bigquery_service() + + jobs = bigquery_service.jobs() + job_data = { + "configuration": { + "query": { + "query": query, + } + } + } + + logging.debug("bigquery got query: %s", query) + + project_id = connection_string["projectId"] + + try: + insert_response = jobs.insert(projectId=project_id, body=job_data).execute() + current_row = 0 + query_reply = jobs.getQueryResults(projectId=project_id, jobId=insert_response['jobReference']['jobId'], startIndex=current_row).execute() + + rows = [] + field_names = [] + for f in query_reply["schema"]["fields"]: + field_names.append(f["name"]) + + while(("rows" in query_reply) and current_row < query_reply['totalRows']): + for row in query_reply["rows"]: + row_data = {} + column_index = 0 + for cell in row["f"]: + row_data[field_names[column_index]] = cell["v"] + column_index += 1 + + rows.append(row_data) + + current_row += len(query_reply['rows']) + query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], startIndex=current_row).execute() + + columns = [{'name': name, + 'friendly_name': name, + 'type': None} for name in field_names] + + data = { + "columns" : columns, + "rows" : rows + } + error = None + + json_data = json.dumps(data, cls=JSONEncoder) + except apiclient.errors.HttpError, e: + json_data = None + error = e.args[1] + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + + + return query_runner \ No newline at end of file diff --git a/redash/settings.py b/redash/settings.py index b06bf4976..09dd127f6 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -50,6 +50,8 @@ CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" # -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" # -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true} +# -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" } +# to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From f54d08a62849965da0bafeb9c74c1c75b6ed62e0 Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Wed, 5 Mar 2014 09:05:33 +0200 Subject: [PATCH 317/540] Added try..except to handle missing imports --- redash/data/query_runner_bigquery.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/redash/data/query_runner_bigquery.py b/redash/data/query_runner_bigquery.py index d6929273b..18150b8d3 100644 --- a/redash/data/query_runner_bigquery.py +++ b/redash/data/query_runner_bigquery.py @@ -3,10 +3,14 @@ import json import logging import sys -import apiclient.errors -from apiclient.discovery import build -from apiclient.errors import HttpError -from oauth2client.client import SignedJwtAssertionCredentials +try: + import apiclient.errors + from apiclient.discovery import build + from apiclient.errors import HttpError + from oauth2client.client import SignedJwtAssertionCredentials +except ImportError: + print "Missing dependencies. Please install google-api-python-client and oauth2client." + print "You can use pip: pip install google-api-python-client oauth2client" from redash.utils import JSONEncoder @@ -91,4 +95,4 @@ def bigquery(connection_string): return json_data, error - return query_runner \ No newline at end of file + return query_runner From 2b03973cf01c994c876ddbf35ff821340e971cee Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Mar 2014 19:18:07 +0200 Subject: [PATCH 318/540] Fix: the pivot table visualization was messing up other visualizations by changing the data. --- rd_ui/app/scripts/query_fiddle/renderers.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/query_fiddle/renderers.js b/rd_ui/app/scripts/query_fiddle/renderers.js index 1eb965e16..00aa55e1c 100644 --- a/rd_ui/app/scripts/query_fiddle/renderers.js +++ b/rd_ui/app/scripts/query_fiddle/renderers.js @@ -16,7 +16,10 @@ renderers.directive('pivotTableRenderer', function () { if ($scope.queryResult.getData() == null) { } else { - $(element).pivotUI($scope.queryResult.getData(), { + // We need to give the pivot table its own copy of the data, because its change + // it which interferes with other visualizations. + var data = $.extend(true, [], $scope.queryResult.getData()); + $(element).pivotUI(data, { renderers: $.pivotUtilities.renderers }, true); } From 5e981a579b645226b3dd0c3df8f6e15f69b32c29 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Mar 2014 19:21:13 +0200 Subject: [PATCH 319/540] Pie chart: show value and not only % --- rd_ui/app/scripts/ng-highchart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 6e4f25b59..7407988e9 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -124,7 +124,7 @@ enabled: true, color: '#000000', connectorColor: '#000000', - format: '{point.name}: {point.percentage:.1f} %' + format: '{point.name}: {point.y} ({point.percentage:.1f} %)' } }, scatter: { From ceb2e0cfb343d99576490d174625bd8d02e3cc5f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Mar 2014 19:23:56 +0200 Subject: [PATCH 320/540] Fix: custom visualization name was ignored (ref #127) --- rd_ui/app/scripts/visualizations/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 7d9e5071b..d940edbe0 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -112,9 +112,9 @@ }, true); } - scope.$watch('visualization.type', function (type) { + scope.$watch('visualization.type', function (type, oldType) { // if not edited by user, set name to match type - if (type && scope.visualization && !scope.visForm.name.$dirty) { + if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) { // poor man's titlecase scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase(); } From db1df073373e1e26273e2edf555dddf4d78d0427 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 6 Mar 2014 19:58:04 +0200 Subject: [PATCH 321/540] Hackish way to show dates as dates in the table and timestamps as date+timestamp. --- rd_ui/app/scripts/visualizations/table.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index 98ee06d99..5b5835e3c 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -63,7 +63,13 @@ columnDefinition['formatParameter'] = 2; } else if (moment.isMoment(exampleData)) { columnDefinition['formatFunction'] = function(value) { - return value.format("DD/MM/YY HH:mm"); + // TODO: this is very hackish way to determine if we need + // to show the value as a time or date only. Better solution + // is to complete #70 and use the information it returns. + if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) { + return value.format("DD/MM/YY HH:mm"); + } + return value.format("DD/MM/YY"); } } } From 30b4628593febdb5feff46f3379e7782825a5c68 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 9 Mar 2014 12:52:33 +0200 Subject: [PATCH 322/540] Fix: compile all views. --- rd_ui/Gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/Gruntfile.js b/rd_ui/Gruntfile.js index 8ed52f3b8..5d9c59df8 100644 --- a/rd_ui/Gruntfile.js +++ b/rd_ui/Gruntfile.js @@ -231,7 +231,7 @@ module.exports = function (grunt) { files: [{ expand: true, cwd: '<%= yeoman.app %>', - src: ['*.html', 'views/*.html'], + src: ['*.html', 'views/**/*.html'], dest: '<%= yeoman.dist %>' }] } From 2cbee1bf8246b0c569f64c45a734bab319e70011 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 11 Mar 2014 15:14:32 +0200 Subject: [PATCH 323/540] Fix: don't sort values when there is more than one category --- rd_ui/app/scripts/ng-highchart.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/ng-highchart.js b/rd_ui/app/scripts/ng-highchart.js index 7407988e9..9bbc2b1ee 100644 --- a/rd_ui/app/scripts/ng-highchart.js +++ b/rd_ui/app/scripts/ng-highchart.js @@ -225,12 +225,16 @@ // TODO: move this logic to Query#getChartData var yValues = _.groupBy(s.data, 'x'); - var newData = _.sortBy(_.map(categories, function (category) { + var newData = _.map(categories, function (category) { return { name: category, y: yValues[category] && yValues[category][0].y } - }), 'y').reverse(); + }); + + if (categories.length == 1) { + newData = _.sortBy(newData, 'y').reverse(); + }; s.data = newData; }); From e57fabbd1d558df264cccc194d840c2c9081e366 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 4 Mar 2014 15:45:26 +0200 Subject: [PATCH 324/540] #121 query view page --- rd_ui/app/scripts/app.js | 5 +++ rd_ui/app/views/queryview.html | 67 ++++++++++++++++++++++++++++++++++ redash/controllers.py | 3 +- 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 rd_ui/app/views/queryview.html diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 52ac7029d..205b65de1 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -35,6 +35,11 @@ angular.module('redash', [ controller: 'QueryFiddleCtrl', reloadOnSearch: false }); + $routeProvider.when('/queries/view/:queryId', { + templateUrl: '/views/queryview.html', + controller: 'QueryFiddleCtrl', + reloadOnSearch: false + }); $routeProvider.when('/queries/:queryId', { templateUrl: '/views/queryfiddle.html', controller: 'QueryFiddleCtrl', diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html new file mode 100644 index 000000000..301e317b0 --- /dev/null +++ b/rd_ui/app/views/queryview.html @@ -0,0 +1,67 @@ + +
      +
      +
      +

      + {{ query.name }} + + Download + + +

      +
      +
      + +
      + +
      +
      +
      + {{ query.description }} +
      +
      +

      + + Last update + +

      +

      + + Created By {{query.user.name}} +

      +

      + + Runtime + {{queryResult.getRuntime() | durationHumanize}} +

      +

      + + Rows {{queryResult.getData().length}} +

      + +
      + +
      +
      +
      +
        + + + + +
      +
      +
      +
      +
      + + + +
      + +
      +
      +
      +
      +
      +
      diff --git a/redash/controllers.py b/redash/controllers.py index 7f8ce455b..9ccd93737 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -33,6 +33,7 @@ def ping(): @app.route('/dashboard/') @app.route('/queries') @app.route('/queries/') +@app.route('/queries/view/') @app.route('/') @auth.required def index(anything=None): @@ -243,7 +244,7 @@ class QueryAPI(BaseResource): query_def['latest_query_data'] = query_def.pop('latest_query_data_id') models.Query.update_instance(query_id, **query_def) - + query = models.Query.get_by_id(query_id) return query.to_dict(with_result=False, with_visualizations=True) From 61316c40e5bf8d8504917d36b2d213841df9e0d3 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 4 Mar 2014 16:17:52 +0200 Subject: [PATCH 325/540] #121: QueryViewCtrl with 'strict mode' --- rd_ui/app/index.html | 1 + rd_ui/app/scripts/app.js | 10 +- rd_ui/app/scripts/controllers/QueryView.js | 262 +++++++++++++++++++++ 3 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 rd_ui/app/scripts/controllers/QueryView.js diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 05145c8c5..260c29081 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -111,6 +111,7 @@ + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 205b65de1..a68f92e20 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -35,16 +35,16 @@ angular.module('redash', [ controller: 'QueryFiddleCtrl', reloadOnSearch: false }); - $routeProvider.when('/queries/view/:queryId', { - templateUrl: '/views/queryview.html', - controller: 'QueryFiddleCtrl', - reloadOnSearch: false - }); $routeProvider.when('/queries/:queryId', { templateUrl: '/views/queryfiddle.html', controller: 'QueryFiddleCtrl', reloadOnSearch: false }); + $routeProvider.when('/queries/view/:queryId', { + templateUrl: '/views/queryview.html', + controller: 'QueryViewCtrl', + reloadOnSearch: false + }); $routeProvider.when('/admin/status', { templateUrl: '/views/admin_status.html', controller: 'AdminStatusCtrl' diff --git a/rd_ui/app/scripts/controllers/QueryView.js b/rd_ui/app/scripts/controllers/QueryView.js new file mode 100644 index 000000000..3b381f433 --- /dev/null +++ b/rd_ui/app/scripts/controllers/QueryView.js @@ -0,0 +1,262 @@ +(function() { + 'use strict'; + + var QueryViewCtrl = function($scope, $window, $routeParams, $http, $location, growl, notifications, Query, Visualization) { + var DEFAULT_TAB = 'table'; + var pristineHash = null; + var leavingPageText = "You will lose your changes if you leave"; + + $scope.dirty = undefined; + $scope.newVisualization = undefined; + + $window.onbeforeunload = function() { + if (currentUser.canEdit($scope.query) && $scope.dirty) { + return leavingPageText; + } + } + + Mousetrap.bindGlobal("meta+s", function(e) { + e.preventDefault(); + + if (currentUser.canEdit($scope.query)) { + $scope.saveQuery(); + } + }); + + $scope.$on('$locationChangeStart', function(event, next, current) { + if (next.split("#")[0] == current.split("#")[0]) { + return; + } + + if (!currentUser.canEdit($scope.query)) { + return; + } + + if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { + event.preventDefault(); + } else { + Mousetrap.unbind("meta+s"); + } + }); + + $scope.$parent.pageTitle = "Query Fiddle"; + + $scope.$watch(function() { + return $location.hash() + }, function(hash) { + $scope.selectedTab = hash || DEFAULT_TAB; + }); + + $scope.lockButton = function(lock) { + $scope.queryExecuting = lock; + }; + + $scope.formatQuery = function() { + $scope.editorOptions.readOnly = 'nocursor'; + + $http.post('/api/queries/format', { + 'query': $scope.query.query + }).success(function(response) { + $scope.query.query = response; + $scope.editorOptions.readOnly = false; + }) + } + + $scope.saveQuery = function(duplicate, oldId) { + if (!oldId) { + oldId = $scope.query.id; + } + + delete $scope.query.latest_query_data; + $scope.query.$save(function(q) { + pristineHash = q.getHash(); + $scope.dirty = false; + + if (duplicate) { + growl.addInfoMessage("Query duplicated.", { + ttl: 2000 + }); + } else { + growl.addSuccessMessage("Query saved.", { + ttl: 2000 + }); + } + + if (oldId != q.id) { + if (oldId == undefined) { + $location.path($location.path().replace('new', q.id)).replace(); + } else { + // TODO: replace this with a safer method + $location.path($location.path().replace(oldId, q.id)).replace(); + + // Reset visualizations tab to table after duplicating a query: + $location.hash('table'); + } + } + }, function(httpResponse) { + growl.addErrorMessage("Query could not be saved"); + }); + }; + + $scope.duplicateQuery = function() { + var oldId = $scope.query.id; + $scope.query.id = null; + $scope.query.ttl = -1; + + $scope.saveQuery(true, oldId); + }; + + // Query Editor: + $scope.editorOptions = { + mode: 'text/x-sql', + lineWrapping: true, + lineNumbers: true, + readOnly: false, + matchBrackets: true, + autoCloseBrackets: true + }; + + $scope.refreshOptions = [{ + value: -1, + name: 'No Refresh' + }, { + value: 60, + name: 'Every minute' + }, ] + + _.each(_.range(1, 13), function(i) { + $scope.refreshOptions.push({ + value: i * 3600, + name: 'Every ' + i + 'h' + }); + }) + + $scope.refreshOptions.push({ + value: 24 * 3600, + name: 'Every 24h' + }); + $scope.refreshOptions.push({ + value: 7 * 24 * 3600, + name: 'Once a week' + }); + + $scope.$watch('queryResult && queryResult.getError()', function(newError, oldError) { + if (newError == undefined) { + return; + } + + if (oldError == undefined && newError != undefined) { + $scope.lockButton(false); + } + }); + + $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) { + if (!data) { + return; + } + + if ($scope.queryResult.getId() == null) { + $scope.dataUri = ""; + } else { + $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv'; + $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv"; + } + }); + + $scope.$watch("queryResult && queryResult.getStatus()", function(status) { + if (!status) { + return; + } + + if (status == "done") { + if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() && + $scope.query.query_hash == $scope.queryResult.query_result.query_hash) { + Query.save({ + 'id': $scope.query.id, + 'latest_query_data_id': $scope.queryResult.getId() + }) + } + $scope.query.latest_query_data_id = $scope.queryResult.getId(); + + notifications.showNotification("re:dash", $scope.query.name + " updated."); + + $scope.lockButton(false); + } + }); + + if ($routeParams.queryId != undefined) { + $scope.query = Query.get({ + id: $routeParams.queryId + }, function(q) { + pristineHash = q.getHash(); + $scope.dirty = false; + $scope.queryResult = $scope.query.getQueryResult(); + }); + } else { + $scope.query = new Query({ + query: "", + name: "New Query", + ttl: -1, + user: currentUser + }); + $scope.lockButton(false); + } + + $scope.$watch('query.name', function() { + $scope.$parent.pageTitle = $scope.query.name; + }); + + $scope.$watch(function() { + return $scope.query.getHash(); + }, function(newHash) { + $scope.dirty = (newHash !== pristineHash); + }); + + $scope.executeQuery = function() { + $scope.queryResult = $scope.query.getQueryResult(0); + $scope.lockButton(true); + $scope.cancelling = false; + }; + + $scope.cancelExecution = function() { + $scope.cancelling = true; + $scope.queryResult.cancelExecution(); + }; + + $scope.deleteVisualization = function($e, vis) { + $e.preventDefault(); + if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { + Visualization.delete(vis); + if ($scope.selectedTab == vis.id) { + $scope.selectedTab = DEFAULT_TAB; + } + $scope.query.visualizations = + $scope.query.visualizations.filter(function(v) { + return vis.id !== v.id; + }); + } + }; + + var unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { + if (newPanel && $routeParams.queryId == undefined) { + unbind(); + $scope.saveQuery(); + } + }); + }; + + angular.module('redash.controllers').controller('QueryViewCtrl', + [ + '$scope', + '$window', + '$routeParams', + '$http', + '$location', + 'growl', + 'notifications', + 'Query', + 'Visualization', + QueryViewCtrl + ]); + +})(); \ No newline at end of file From e0c330fb291a98a4d5386f5703a5f755d53ff252 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 5 Mar 2014 14:05:53 +0200 Subject: [PATCH 326/540] #121: QueryView page edit mode --- rd_ui/app/scripts/controllers/QueryView.js | 10 +++ rd_ui/app/views/queryview.html | 71 ++++++++++++++++++---- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/rd_ui/app/scripts/controllers/QueryView.js b/rd_ui/app/scripts/controllers/QueryView.js index 3b381f433..387e0a1cf 100644 --- a/rd_ui/app/scripts/controllers/QueryView.js +++ b/rd_ui/app/scripts/controllers/QueryView.js @@ -6,9 +6,13 @@ var pristineHash = null; var leavingPageText = "You will lose your changes if you leave"; + $scope.isEditing = false; $scope.dirty = undefined; $scope.newVisualization = undefined; + $scope.queryExecuting = false; + $scope.queryResultStatus = null; + $window.onbeforeunload = function() { if (currentUser.canEdit($scope.query) && $scope.dirty) { return leavingPageText; @@ -47,6 +51,10 @@ $scope.selectedTab = hash || DEFAULT_TAB; }); + $scope.toggleEdit = function (state) { + $scope.isEditing = (state !== undefined) ? state : !$scope.isEditing; + }; + $scope.lockButton = function(lock) { $scope.queryExecuting = lock; }; @@ -164,6 +172,8 @@ }); $scope.$watch("queryResult && queryResult.getStatus()", function(status) { + $scope.queryResultStatus = status; + if (!status) { return; } diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index 301e317b0..43b014947 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -3,10 +3,21 @@

      - {{ query.name }} + Editing {{ query.name }} - Download - + + + +

      @@ -15,14 +26,42 @@
      -
      +
      +
      +

      + + +

      + +

      +

      + Executing query… + Query in queue… + + +
      +
      Error running query: {{queryResult.getError()}}
      +

      + +

      + + +

      +
      +
      {{ query.description }}

      - Last update + Last update

      @@ -31,24 +70,26 @@

      - Runtime - {{queryResult.getRuntime() | durationHumanize}} + Runtime + {{queryResult.getRuntime() | durationHumanize}} + Running…

      Rows {{queryResult.getData().length}}

      -
      -
      +
        - - + + × + +
      @@ -57,8 +98,14 @@ -
      +
      + +
      + +
      + +
      From 51db8346d303634c49d711a28d8968a6b4147656 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 6 Mar 2014 11:42:02 +0200 Subject: [PATCH 327/540] organizing /app files --- rd_ui/app/index.html | 22 ++++++++++--------- .../{ => controllers}/admin_controllers.js | 0 .../scripts/{ => controllers}/controllers.js | 0 .../{QueryView.js => query_view.js} | 0 .../{ng-highchart.js => ng_highchart.js} | 0 .../{smart-table.js => ng_smart_table.js} | 0 rd_ui/app/scripts/{ => services}/services.js | 0 .../renderers.js => visualizations/pivot.js} | 0 8 files changed, 12 insertions(+), 10 deletions(-) rename rd_ui/app/scripts/{ => controllers}/admin_controllers.js (100%) rename rd_ui/app/scripts/{ => controllers}/controllers.js (100%) rename rd_ui/app/scripts/controllers/{QueryView.js => query_view.js} (100%) rename rd_ui/app/scripts/{ng-highchart.js => ng_highchart.js} (100%) rename rd_ui/app/scripts/{smart-table.js => ng_smart_table.js} (100%) rename rd_ui/app/scripts/{ => services}/services.js (100%) rename rd_ui/app/scripts/{query_fiddle/renderers.js => visualizations/pivot.js} (100%) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 260c29081..ae2d63465 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -97,32 +97,34 @@ - - - + + + + + - - + + + + + + - + - - - - diff --git a/rd_ui/app/scripts/services/dashboards.js b/rd_ui/app/scripts/services/dashboards.js index 1eb0c08e3..2b515572a 100644 --- a/rd_ui/app/scripts/services/dashboards.js +++ b/rd_ui/app/scripts/services/dashboards.js @@ -2,7 +2,7 @@ var Dashboard = function($resource) { var resource = $resource('/api/dashboards/:slug', {slug: '@slug'}); resource.prototype.canEdit = function() { - return currentUser.is_admin || currentUser.canEdit(this); + return currentUser.hasPermission('admin') || currentUser.canEdit(this); } return resource; } diff --git a/rd_ui/app/views/index.html b/rd_ui/app/views/index.html index 8db531646..6fb42995d 100644 --- a/rd_ui/app/views/index.html +++ b/rd_ui/app/views/index.html @@ -3,7 +3,7 @@
      {{name}} - +
      @@ -11,7 +11,7 @@
      -
      +
      Admin
      Status diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index bfbf92e33..db0b90abc 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -13,7 +13,7 @@

      -
      +
      re:dash + {{name}}
      {% raw %}
      diff --git a/rd_ui/app/login.html b/rd_ui/app/login.html index a5271f9d4..8088cdfc3 100644 --- a/rd_ui/app/login.html +++ b/rd_ui/app/login.html @@ -4,7 +4,7 @@ - re:dash Login + {{name}} Login @@ -26,7 +26,7 @@ - re:dash + {{name}}
      diff --git a/redash/controllers.py b/redash/controllers.py index 2c61243f8..21e76edc4 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -50,7 +50,8 @@ def index(**kwargs): 'permissions': current_user.permissions } - return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS) + return render_template("index.html", user=json.dumps(user), name=settings.NAME, + analytics=settings.ANALYTICS) @app.route('/login', methods=['GET', 'POST']) @@ -70,6 +71,7 @@ def login(): return redirect(request.args.get('next') or '/') return render_template("login.html", + name=settings.NAME, analytics=settings.ANALYTICS, next=request.args.get('next'), username=request.form.get('username', ''), diff --git a/redash/settings.py b/redash/settings.py index 09dd127f6..b454afa73 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -44,6 +44,8 @@ STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1") STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125")) STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash") +NAME = os.environ.get('REDASH_NAME', 're:dash') + # "pg", "graphite" or "mysql" CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # Connection string for the database that is used to run queries against. Examples: From 0e3c6ac275a37b03c88e6f89a529a1b41ebcf3fa Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Mar 2014 13:32:28 +0200 Subject: [PATCH 367/540] Create table only if it doesn't exists --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index 1ff20c320..b52165314 100644 --- a/redash/models.py +++ b/redash/models.py @@ -322,7 +322,7 @@ def create_db(create_tables, drop_tables): db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table) #model.drop_table() - if create_tables: + if create_tables and not model.table_exists(): model.create_table() db.close_db(None) \ No newline at end of file From 607123e67a59e95ea224ae44afe67ecba32acb63 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Mar 2014 13:41:42 +0200 Subject: [PATCH 368/540] Reduce expire time to 1800, to reduce changes of test failing --- tests/test_controllers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 47453165a..19aaeb876 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -339,7 +339,7 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def test_returns_200_for_correct_signature(self): with app.test_client() as c: - expires = time.time()+3600 + expires = time.time()+1800 rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) self.assertEquals(rv.status_code, 200) From 3b11f010b514ba913fc77f6c540de87c7dec56ad Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 12 Mar 2014 16:53:22 +0200 Subject: [PATCH 369/540] Fix minification issue --- rd_ui/app/scripts/app.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index f59c3528e..a3bf63bb3 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -27,7 +27,7 @@ angular.module('redash', [ }); return defer.promise; - } + }; $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/); $locationProvider.html5Mode(true); @@ -59,7 +59,7 @@ angular.module('redash', [ controller: 'QueryViewCtrl', reloadOnSearch: false, resolve: { - 'query': getQuery + 'query': ['Query', '$q', '$route', getQuery] } }); $routeProvider.when('/queries/:queryId/fiddle', { @@ -72,7 +72,7 @@ angular.module('redash', [ controller: 'QueryViewCtrl', reloadOnSearch: false, resolve: { - 'query': getQuery, + 'query': ['Query', '$q', '$route', getQuery], 'viewSource': function isViewSource() { return true; } From 97d614659a4ed69e38b55fd0ea6350ce3d782d4e Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 12 Mar 2014 17:31:01 +0200 Subject: [PATCH 370/540] logout button. closes #125 --- rd_ui/app/index.html | 7 +++++-- rd_ui/app/styles/redash.css | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index ae2d63465..092613b0d 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -64,8 +64,11 @@
      diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 4dedc34ab..d8a48dab0 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -25,6 +25,17 @@ a.navbar-brand { margin-top: 5px; margin-bottom: 5px; } +.avatar img { + width: 40px; + height: 40px; +} + +#logout { + color: white; + position: relative; + left: -9px; + bottom: -11px; +} .edit-in-place span { white-space: pre-line; From d435d122eb26142220d1a42637a0e86e9e9eacd3 Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 16:36:51 +0200 Subject: [PATCH 371/540] Added support for running scripts as queries --- redash/data/manager.py | 3 ++ redash/data/query_runner_script.py | 51 ++++++++++++++++++++++++++++++ redash/settings.py | 5 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 redash/data/query_runner_script.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 55a10066d..93499eed4 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -185,6 +185,9 @@ class Manager(object): from redash.data import query_runner_bigquery connection_params = json.loads(connection_string) runner = query_runner_bigquery.bigquery(connection_params) + elif connection_type == 'script': + from redash.data import query_runner_script + runner = query_runner_script.script(connection_string) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_script.py b/redash/data/query_runner_script.py new file mode 100644 index 000000000..902bd6f0e --- /dev/null +++ b/redash/data/query_runner_script.py @@ -0,0 +1,51 @@ +import json +import logging +import sys +import os +import subprocess + +# We use subprocess.check_output because we are lazy. +# If someone will really want to run this on Python < 2.7 they can easily update the code to run +# Popen, check the retcodes and other things and read the standard output to a variable. +if not "check_output" in subprocess.__dict__: + print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7" + +def script(connection_string): + + def query_runner(query): + try: + json_data = None + error = None + + # Remove the SQL comment that Redash adds + if query.find("/*") > -1 and query.find("*/") > -1: + query = query[query.find("*/")+3:] + + # Poor man's protection against running scripts from output the scripts directory + if connection_string.find("../") > -1: + return None, "Scripts can only be run from the configured scripts directory" + + query = query.strip() + + script = os.path.join(connection_string, query) + if not os.path.exists(script): + return None, "Script '%s' not found in script directory" % query + + output = subprocess.check_output(script, shell=False) + if output != None: + output = output.strip() + if output != "": + return output, None + + error = "Error reading output" + except subprocess.CalledProcessError as e: + return None, str(e) + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + + return query_runner diff --git a/redash/settings.py b/redash/settings.py index b454afa73..d28575eff 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -46,7 +46,7 @@ STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash") NAME = os.environ.get('REDASH_NAME', 're:dash') -# "pg", "graphite" or "mysql" +# "pg", "graphite", "mysql", "bigquery" or "script" CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # Connection string for the database that is used to run queries against. Examples: # -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" @@ -54,6 +54,9 @@ CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true} # -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" } # to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts +# -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS" (.i.e /home/user/redash_scripts/) +# all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING. +# The output of the scripts must be in the output format defined here: CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From d3e87a3d28bd010d0e45a8733d0b189caebe8c8c Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 18:44:31 +0200 Subject: [PATCH 372/540] added support for a 'url' source where you can supply a URL to retrieve the same JSON result used in other query runners --- redash/data/manager.py | 3 +++ redash/data/query_runner_url.py | 48 +++++++++++++++++++++++++++++++++ redash/settings.py | 2 ++ 3 files changed, 53 insertions(+) create mode 100644 redash/data/query_runner_url.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 93499eed4..5530c76a1 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -188,6 +188,9 @@ class Manager(object): elif connection_type == 'script': from redash.data import query_runner_script runner = query_runner_script.script(connection_string) + elif connection_type == 'url': + from redash.data import query_runner_url + runner = query_runner_url.url(connection_string) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_url.py b/redash/data/query_runner_url.py new file mode 100644 index 000000000..1cb57f17b --- /dev/null +++ b/redash/data/query_runner_url.py @@ -0,0 +1,48 @@ +import json +import logging +import sys +import os +import urllib2 + +def url(connection_string): + + def query_runner(query): + base_url = connection_string + + try: + json_data = None + error = None + + query = query.strip() + + # Remove the SQL comment that Redash adds + if query.find("/*") > -1 and query.find("*/") > -1: + query = query[query.find("*/")+3:] + + if base_url is not None and base_url != "": + if query.find("://") > -1: + return None, "Accepting only relative URLs to '%s'" % base_url + + if base_url is None: + base_url = "" + + url = base_url + query + + json_data = urllib2.urlopen(url).read().strip() + + if not json_data: + error = "Error reading data from '%s'" % url + + return json_data, error + + except urllib2.URLError as e: + return None, str(e) + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + + return query_runner diff --git a/redash/settings.py b/redash/settings.py index d28575eff..a931085bf 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -57,6 +57,8 @@ CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS" (.i.e /home/user/redash_scripts/) # all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING. # The output of the scripts must be in the output format defined here: +# -- url: CONNECTION_STRING = "base URL" (i.e. http://myserver/somewhere) +# If CONNECTION_STRING is set, the query should be a relative URL. If it is not set a full URL can be used CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From a2257999a7bf2fde4ec084ac974619160c0c5e4d Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 18:56:50 +0200 Subject: [PATCH 373/540] moved to use the query_runner.annotate_query flag so we won't get the SQL comment --- redash/data/query_runner_script.py | 5 +---- redash/data/query_runner_url.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/redash/data/query_runner_script.py b/redash/data/query_runner_script.py index 902bd6f0e..dbee7c780 100644 --- a/redash/data/query_runner_script.py +++ b/redash/data/query_runner_script.py @@ -17,10 +17,6 @@ def script(connection_string): json_data = None error = None - # Remove the SQL comment that Redash adds - if query.find("/*") > -1 and query.find("*/") > -1: - query = query[query.find("*/")+3:] - # Poor man's protection against running scripts from output the scripts directory if connection_string.find("../") > -1: return None, "Scripts can only be run from the configured scripts directory" @@ -48,4 +44,5 @@ def script(connection_string): return json_data, error + query_runner.annotate_query = False return query_runner diff --git a/redash/data/query_runner_url.py b/redash/data/query_runner_url.py index 1cb57f17b..64f146bd5 100644 --- a/redash/data/query_runner_url.py +++ b/redash/data/query_runner_url.py @@ -15,10 +15,6 @@ def url(connection_string): query = query.strip() - # Remove the SQL comment that Redash adds - if query.find("/*") > -1 and query.find("*/") > -1: - query = query[query.find("*/")+3:] - if base_url is not None and base_url != "": if query.find("://") > -1: return None, "Accepting only relative URLs to '%s'" % base_url @@ -45,4 +41,5 @@ def url(connection_string): return json_data, error + query_runner.annotate_query = False return query_runner From 19209d16aa0dbc2a6b27356096dab18db569eaa9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:20:49 +0200 Subject: [PATCH 374/540] Filters directive. --- rd_ui/app/scripts/visualizations/base.js | 8 ++++++++ rd_ui/app/views/grid_renderer.html | 13 ------------- rd_ui/app/views/visualizations/filters.html | 12 ++++++++++++ 3 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 rd_ui/app/views/visualizations/filters.html diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index d940edbe0..2d6061c71 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -77,6 +77,13 @@ } }; + var Filters = function() { + return { + restrict: 'E', + templateUrl: '/views/visualizations/filters.html' + } + } + var EditVisualizationForm = function(Visualization, growl) { return { restrict: 'E', @@ -145,5 +152,6 @@ .provider('Visualization', VisualizationProvider) .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor]) + .directive('filters', Filters) .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm]) })(); diff --git a/rd_ui/app/views/grid_renderer.html b/rd_ui/app/views/grid_renderer.html index 3524c8338..148e9c8f5 100644 --- a/rd_ui/app/views/grid_renderer.html +++ b/rd_ui/app/views/grid_renderer.html @@ -1,17 +1,4 @@
      -
      - - -
      - diff --git a/rd_ui/app/views/visualizations/filters.html b/rd_ui/app/views/visualizations/filters.html new file mode 100644 index 000000000..3d718a944 --- /dev/null +++ b/rd_ui/app/views/visualizations/filters.html @@ -0,0 +1,12 @@ +
      +
      + + +
      +
      \ No newline at end of file From 2b5291900daa8fe19b40210fd58f28e29644bb76 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:21:39 +0200 Subject: [PATCH 375/540] Show filters directive on all visualizations. --- rd_ui/app/scripts/visualizations/base.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 2d6061c71..5a0a34daf 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -64,8 +64,11 @@ // TODO: using switch here (and in the options editor) might introduce errors and bad // performance wise. It's better to eventually show the correct template based on the // visualization type and not make the browser render all of them. - template: Visualization.renderVisualizationsTemplate, - replace: false + template: '\n' + Visualization.renderVisualizationsTemplate, + replace: false, + link: function(scope) { + scope.filters = scope.queryResult.getFilters(); + } } }; @@ -148,6 +151,8 @@ } }; + + angular.module('redash.visualization', []) .provider('Visualization', VisualizationProvider) .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) From a99e290bc5c77c540f2245f48af22c6cbf274861 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:22:08 +0200 Subject: [PATCH 376/540] Store filters on QueryResult object and use them in getQueryData. --- rd_ui/app/scripts/services/services.js | 56 ++++++++++++++++++++--- rd_ui/app/scripts/visualizations/table.js | 42 +++++++---------- 2 files changed, 67 insertions(+), 31 deletions(-) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 9a4f5d870..3a1cac780 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -7,6 +7,7 @@ angular.extend(this, props); if ('query_result' in props) { this.status = "done"; + this.filters = undefined; _.each(this.query_result.data.rows, function (row) { _.each(row, function (v, k) { @@ -26,6 +27,7 @@ this.job = {}; this.query_result = {}; this.status = "waiting"; + this.filters = undefined; this.updatedAt = moment(); @@ -76,15 +78,51 @@ return this.query_result.runtime; } - QueryResult.prototype.getData = function() { + QueryResult.prototype.getRawData = function() { if (!this.query_result.data) { return null; } var data = this.query_result.data.rows; + return data; } + QueryResult.prototype.getData = function() { + if (!this.query_result.data) { + return null; + } + + var filterValues = function(filters) { + if (!filters) { + return null; + } + + return _.reduce(filters, function(str, filter) { + return str + filter.current; + }, "") + } + + var filters = this.getFilters(); + var filterFreeze = filterValues(filters); + + if (this.filterFreeze != filterFreeze) { + this.filterFreeze = filterFreeze; + + if (filters) { + this.filteredData = _.filter(this.query_result.data.rows, function (row) { + return _.reduce(filters, function (memo, filter) { + return (memo && row[filter.name] == filter.current); + }, true); + }); + } else { + this.filteredData = this.query_result.data.rows; + } + } + + return this.filteredData; + } + QueryResult.prototype.getChartData = function () { var series = {}; @@ -181,6 +219,14 @@ } QueryResult.prototype.getFilters = function () { + if (!this.filters) { + this.prepareFilters(); + } + + return this.filters; + }; + + QueryResult.prototype.prepareFilters = function() { var filterNames = []; _.each(this.getColumns(), function (col) { if (col.split('::')[1] == 'filter') { @@ -189,7 +235,7 @@ }); var filterValues = []; - _.each(this.getData(), function (row) { + _.each(this.getRawData(), function (row) { _.each(filterNames, function (filter, i) { if (filterValues[i] == undefined) { filterValues[i] = []; @@ -198,7 +244,7 @@ }) }); - var filters = _.map(filterNames, function (filter, i) { + this.filters = _.map(filterNames, function (filter, i) { var f = { name: filter, friendlyName: this.getColumnFriendlyName(filter), @@ -208,9 +254,7 @@ f.current = f.values[0]; return f; }, this); - - return filters; - }; + } var refreshStatus = function(queryResult, query, ttl) { Job.get({'id': queryResult.job.id}, function(response) { diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index 5b5835e3c..70dfd0005 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -38,17 +38,21 @@ $scope.gridData = []; $scope.filters = []; } else { - - $scope.filters = $scope.queryResult.getFilters(); - var gridData = _.map($scope.queryResult.getData(), function (row) { - var newRow = {}; - _.each(row, function (val, key) { - newRow[$scope.queryResult.getColumnCleanName(key)] = val; - }) - return newRow; - }); + var prepareGridData = function(data) { + var gridData = _.map(data, function (row) { + var newRow = {}; + _.each(row, function (val, key) { + newRow[$scope.queryResult.getColumnCleanName(key)] = val; + }) + return newRow; + }); + + return gridData; + }; + + $scope.gridData = prepareGridData($scope.queryResult.getData()); $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { var columnDefinition = { @@ -56,8 +60,10 @@ 'map': col }; - if (gridData.length > 0) { - var exampleData = gridData[0][col]; + var rawData = $scope.queryResult.getRawData(); + + if (rawData.length > 0) { + var exampleData = rawData[0][col]; if (angular.isNumber(exampleData)) { columnDefinition['formatFunction'] = 'number'; columnDefinition['formatParameter'] = 2; @@ -76,20 +82,6 @@ return columnDefinition; }); - - $scope.gridData = _.clone(gridData); - - $scope.$watch('filters', function (filters) { - $scope.gridData = _.filter(gridData, function (row) { - return _.reduce(filters, function (memo, filter) { - if (filter.current == 'All') { - return memo && true; - } - - return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current); - }, true); - }); - }, true); } }); }] From 44621e4f37db4a36f5359b227d04e8ffa1aeeab8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:22:49 +0200 Subject: [PATCH 377/540] Switch to $watchCollection to resolve the issue of chart not updating when length of series stays the same. --- rd_ui/app/scripts/ng_highchart.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index 9bbc2b1ee..8aed056e7 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -181,17 +181,13 @@ }, true); //Update when charts data changes - scope.$watch(function () { - // TODO: this might be an issue in case the series change, but they stay - // with the same length - return (scope.series && scope.series.length) || 0; - }, function (length) { - if (!length || length == 0) { + scope.$watchCollection('series', function (series) { + if (!series || series.length == 0) { scope.chart.showLoading(); } else { drawChart(); }; - }, true); + }); }); function initChart(options) { From f3d46355afdc5bb78cfd3bf62ccb698fec00cb68 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:23:12 +0200 Subject: [PATCH 378/540] Because we draw the table without VisualizationRenderer we need to explicitly add filters here too. --- rd_ui/app/scripts/controllers/query_view.js | 2 ++ rd_ui/app/views/queryview.html | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 93e8c2ea0..63a737977 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -168,6 +168,8 @@ return; } + $scope.filters = $scope.queryResult.getFilters(); + if ($scope.queryResult.getId() == null) { $scope.dataUri = ""; } else { diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index db0b90abc..0e584b24b 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -150,7 +150,11 @@
      - +
      + + +
      +
      From 9a04535e6b6ab33e1f62a9064ec4ec7ec8c36819 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 20:25:25 +0200 Subject: [PATCH 379/540] Reset filterFreeze when updating data. --- rd_ui/app/scripts/services/services.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 3a1cac780..8d95659a4 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -8,6 +8,7 @@ if ('query_result' in props) { this.status = "done"; this.filters = undefined; + this.filterFreeze = undefined; _.each(this.query_result.data.rows, function (row) { _.each(row, function (v, k) { @@ -28,6 +29,7 @@ this.query_result = {}; this.status = "waiting"; this.filters = undefined; + this.filterFreeze = undefined; this.updatedAt = moment(); From aa7e010342f9e015714466c99eeb3d1c7fde5664 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 21:22:07 +0200 Subject: [PATCH 380/540] Fix: when having categories chart and not all series had values it wouldn't draw the chart with stacking other than none. --- rd_ui/app/scripts/ng_highchart.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index 8aed056e7..d4a38d877 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -224,7 +224,7 @@ var newData = _.map(categories, function (category) { return { name: category, - y: yValues[category] && yValues[category][0].y + y: (yValues[category] && yValues[category][0].y) || 0 } }); @@ -234,6 +234,8 @@ s.data = newData; }); + + console.log(scope.series); } } else { scope.chart.xAxis[0].update({type: 'datetime'}); From cdac5fbf523640eed62c8cc2d15c36326f72f357 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 21:24:18 +0200 Subject: [PATCH 381/540] Remove console.log --- rd_ui/app/scripts/ng_highchart.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index d4a38d877..09ef05280 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -234,8 +234,6 @@ s.data = newData; }); - - console.log(scope.series); } } else { scope.chart.xAxis[0].update({type: 'datetime'}); From b97c9ee3c9c65ecd448c20dbc70349e74d26b1f8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 17 Mar 2014 21:28:48 +0200 Subject: [PATCH 382/540] Feature: import query from json file --- redash/import_export.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/redash/import_export.py b/redash/import_export.py index 2c47469d4..889850701 100644 --- a/redash/import_export.py +++ b/redash/import_export.py @@ -1,3 +1,4 @@ +import contextlib import json from redash import models from flask.ext.script import Manager @@ -108,18 +109,38 @@ class Importer(object): import_manager = Manager(help="import utilities") export_manager = Manager(help="export utilities") -@import_manager.command -def dashboard(mapping_filename, dashboard_filename, user_id): + +@contextlib.contextmanager +def importer_with_mapping_file(mapping_filename): with open(mapping_filename) as f: mapping = json.loads(f.read()) - user = models.User.get_by_id(user_id) - with open(dashboard_filename) as f: - dashboard = json.loads(f.read()) - - importer = Importer(object_mapping=mapping) - importer.import_dashboard(user, dashboard) + importer = Importer(object_mapping=mapping) + yield importer with open(mapping_filename, 'w') as f: f.write(json.dumps(importer.object_mapping, indent=2)) +@import_manager.command +def query(mapping_filename, query_filename, user_id): + user = models.User.get_by_id(user_id) + with open(query_filename) as f: + query = json.loads(f.read()) + + with importer_with_mapping_file(mapping_filename) as importer: + imported_query = importer.import_query(user, query) + + print "New query id: {}".format(imported_query.id) + + +@import_manager.command +def dashboard(mapping_filename, dashboard_filename, user_id): + user = models.User.get_by_id(user_id) + with open(dashboard_filename) as f: + dashboard = json.loads(f.read()) + + with importer_with_mapping_file(mapping_filename) as importer: + importer.import_dashboard(user, dashboard) + + + From c7ded66057662425dc4659f738061305156b873a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 18 Mar 2014 11:25:37 +0200 Subject: [PATCH 383/540] Data sources model --- migrations/create_data_sources.py | 40 +++++++++++++++++++++++++++++++ redash/models.py | 14 ++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 migrations/create_data_sources.py diff --git a/migrations/create_data_sources.py b/migrations/create_data_sources.py new file mode 100644 index 000000000..353f0d5ee --- /dev/null +++ b/migrations/create_data_sources.py @@ -0,0 +1,40 @@ +import logging +import peewee +from playhouse.migrate import Migrator +from redash import db +from redash import models +from redash import settings + +if __name__ == '__main__': + channel = logging.StreamHandler() + logging.getLogger().addHandler(channel) + logging.getLogger().setLevel(settings.LOG_LEVEL) + + db.connect_db() + + if not models.DataSource.table_exists(): + print "Creating data_sources table..." + models.DataSource.create_table() + + default_data_source = models.DataSource.create(name="Default", + type=settings.CONNECTION_ADAPTER, + options=settings.CONNECTION_STRING) + else: + default_data_source = models.DataSource.select().first() + + migrator = Migrator(db.database) + models.Query.data_source.null = True + try: + with db.database.transaction(): + migrator.add_column(models.Query, models.Query.data_source, "data_source_id") + except peewee.ProgrammingError: + print "Failed to create data_source_id column -- assuming it already exists" + + print "Updating data source to existing one..." + models.Query.update(data_source=default_data_source.id).execute() + + with db.database.transaction(): + print "Setting data source to non nullable..." + migrator.set_nullable(models.Query, models.Query.data_source, False) + + db.close_db(None) \ No newline at end of file diff --git a/redash/models.py b/redash/models.py index b52165314..cc0d99a17 100644 --- a/redash/models.py +++ b/redash/models.py @@ -59,7 +59,7 @@ class ActivityLog(BaseModel): id = peewee.PrimaryKeyField() user = peewee.ForeignKeyField(User) - type = peewee.IntegerField() # 1 for query execution + type = peewee.IntegerField() activity = peewee.TextField() created_at = peewee.DateTimeField(default=datetime.datetime.now) @@ -104,8 +104,20 @@ class QueryResult(BaseModel): return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) +class DataSource(BaseModel): + id = peewee.PrimaryKeyField() + name = peewee.CharField() + type = peewee.CharField() + options = peewee.TextField() + created_at = peewee.DateTimeField(default=datetime.datetime.now) + + class Meta: + db_table = 'data_sources' + + class Query(BaseModel): id = peewee.PrimaryKeyField() + data_source = peewee.ForeignKeyField(DataSource) latest_query_data = peewee.ForeignKeyField(QueryResult, null=True) name = peewee.CharField(max_length=255) description = peewee.CharField(max_length=4096, null=True) From caa198964c286e9a1c85e1ecb4ee17f25c400cb9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 18 Mar 2014 11:28:38 +0200 Subject: [PATCH 384/540] Move logging setup to __init__.py so it's always available --- manage.py | 4 ---- migrations/create_data_sources.py | 4 ---- redash/__init__.py | 4 ++++ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/manage.py b/manage.py index f4f2da7e8..3f9ff11cf 100755 --- a/manage.py +++ b/manage.py @@ -101,8 +101,4 @@ manager.add_command("users", users_manager) manager.add_command("import", import_manager) if __name__ == '__main__': - channel = logging.StreamHandler() - logging.getLogger().addHandler(channel) - logging.getLogger().setLevel(settings.LOG_LEVEL) - manager.run() \ No newline at end of file diff --git a/migrations/create_data_sources.py b/migrations/create_data_sources.py index 353f0d5ee..52448a03f 100644 --- a/migrations/create_data_sources.py +++ b/migrations/create_data_sources.py @@ -6,10 +6,6 @@ from redash import models from redash import settings if __name__ == '__main__': - channel = logging.StreamHandler() - logging.getLogger().addHandler(channel) - logging.getLogger().setLevel(settings.LOG_LEVEL) - db.connect_db() if not models.DataSource.table_exists(): diff --git a/redash/__init__.py b/redash/__init__.py index 0f2af7d57..38c650202 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -1,5 +1,6 @@ import json import urlparse +import logging from flask import Flask, make_response from flask.ext.restful import Api from flask_peewee.db import Database @@ -10,6 +11,9 @@ from redash import settings, utils __version__ = '0.3.5' +logging.getLogger().addHandler(logging.StreamHandler()) +logging.getLogger().setLevel(settings.LOG_LEVEL) + app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, static_folder=settings.STATIC_ASSETS_PATH, From 0b994de531b02fb43db298b22274849cb0e62083 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 18 Mar 2014 15:47:29 +0200 Subject: [PATCH 385/540] Refactor Job class to be easier to extend. Moved the Redis logic out of it. --- redash/data/manager.py | 2 +- redash/data/worker.py | 178 +++++++++++++++++++++++++---------------- tests/test_job.py | 90 +++++++++++++++++++++ 3 files changed, 202 insertions(+), 68 deletions(-) create mode 100644 tests/test_job.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 5530c76a1..08532d3b3 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -83,7 +83,7 @@ class Manager(object): logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id) job = worker.Job.load(self.redis_connection, job_id) else: - job = worker.Job(self.redis_connection, query, priority) + job = worker.Job(self.redis_connection, query=query, priority=priority) pipe.multi() job.save(pipe) logging.info("[Manager][%s] Created new job: %s", query_hash, job.id) diff --git a/redash/data/worker.py b/redash/data/worker.py index 00b0f01ae..1c82d23de 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -15,7 +15,71 @@ from statsd import StatsClient from redash.utils import gen_query_hash from redash import settings -class Job(object): + +class RedisObject(object): + # The following should be overriden in the inheriting class: + fields = {} + conversions = {} + id_field = '' + name = '' + + def __init__(self, redis_connection, **kwargs): + self.redis_connection = redis_connection + self.values = {} + + if not self.fields: + raise ValueError("You must set the fields dictionary, before using RedisObject.") + + if not self.name: + raise ValueError("You must set the name, before using RedisObject") + + self.update(**kwargs) + + def __getattr__(self, name): + if name in self.values: + return self.values[name] + else: + raise AttributeError + + def update(self, **kwargs): + for field, default_value in self.fields.iteritems(): + value = kwargs.get(field, self.values.get(field, default_value)) + if callable(value): + value = value() + + if value == 'None': + value = None + + if field in self.conversions and value: + value = self.conversions[field](value) + + self.values[field] = value + + @classmethod + def _redis_key(cls, object_id): + return '{}:{}'.format(cls.name, object_id) + + def save(self, pipe): + if not pipe: + pipe = self.redis_connection.pipeline() + + pipe.sadd('{}_set'.format(self.name), self.id) + pipe.hmset(self._redis_key(self.id), self.values) + pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict())) + + pipe.execute() + + @classmethod + def load(cls, redis_connection, object_id): + object_dict = redis_connection.hgetall(cls._redis_key(object_id)) + obj = None + if object_dict: + obj = cls(redis_connection, **object_dict) + + return obj + + +class Job(RedisObject): HIGH_PRIORITY = 1 LOW_PRIORITY = 2 @@ -24,37 +88,39 @@ class Job(object): DONE = 3 FAILED = 4 - def __init__(self, redis_connection, query, priority, - job_id=None, - wait_time=None, query_time=None, - updated_at=None, status=None, error=None, query_result_id=None, - process_id=0): - self.redis_connection = redis_connection - self.query = query - self.priority = priority - self.query_hash = gen_query_hash(self.query) - self.query_result_id = query_result_id - if process_id == 'None': - self.process_id = None - else: - self.process_id = int(process_id) + fields = { + 'id': lambda: str(uuid.uuid1()), + 'query': None, + 'priority': None, + 'query_hash': None, + 'wait_time': 0, + 'query_time': 0, + 'error': None, + 'updated_at': time.time, + 'status': WAITING, + 'process_id': None, + 'query_result_id': None + } - if job_id is None: - self.id = str(uuid.uuid1()) - self.new_job = True - self.wait_time = 0 - self.query_time = 0 - self.error = None - self.updated_at = time.time() # job_dict.get('updated_at', time.time()) - self.status = self.WAITING # int(job_dict.get('status', self.WAITING)) - else: - self.id = job_id - self.new_job = False - self.error = error - self.wait_time = wait_time - self.query_time = query_time - self.updated_at = updated_at - self.status = status + conversions = { + 'query': lambda query: query.decode('utf-8'), + 'priority': int, + 'updated_at': float, + 'status': int, + 'wait_time': float, + 'query_time': float, + 'process_id': int, + 'query_result_id': int + } + + name = 'job' + + def __init__(self, redis_connection, query, priority, **kwargs): + kwargs['query'] = query + kwargs['priority'] = priority + kwargs['query_hash'] = gen_query_hash(kwargs['query']) + self.new_job = 'id' not in kwargs + super(Job, self).__init__(redis_connection, **kwargs) def to_dict(self): return { @@ -70,10 +136,6 @@ class Job(object): 'process_id': self.process_id } - @staticmethod - def _redis_key(job_id): - return 'job:%s' % job_id - def cancel(self): # TODO: Race condition: # it's possible that it will be picked up by worker while processing the cancel order @@ -95,16 +157,14 @@ class Job(object): if self.is_finished(): pipe.delete('query_hash_job:%s' % self.query_hash) - pipe.sadd('jobs_set', self.id) - pipe.hmset(self._redis_key(self.id), self.to_dict()) - pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict())) - pipe.execute() + super(Job, self).save(pipe) def processing(self, process_id): - self.status = self.PROCESSING - self.process_id = process_id - self.wait_time = time.time() - self.updated_at - self.updated_at = time.time() + self.update(status=self.PROCESSING, + process_id=process_id, + wait_time=time.time() - self.updated_at, + updated_at=time.time()) + self.save() def is_finished(self): @@ -112,37 +172,21 @@ class Job(object): def done(self, query_result_id, error): if error: - self.status = self.FAILED + new_status = self.FAILED else: - self.status = self.DONE + new_status = self.DONE + + self.update(status=new_status, + query_result_id=query_result_id, + error=error, + query_time=time.time() - self.updated_at, + updated_at=time.time()) - self.query_result_id = query_result_id - self.error = error - self.query_time = time.time() - self.updated_at - self.updated_at = time.time() self.save() def __str__(self): return "" % (self.id, self.priority, self.status) - @classmethod - def _load(cls, redis_connection, job_id): - return redis_connection.hgetall(cls._redis_key(job_id)) - - @classmethod - def load(cls, redis_connection, job_id): - job_dict = cls._load(redis_connection, job_id) - job = None - if job_dict: - job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'), - priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']), - status=int(job_dict['status']), wait_time=float(job_dict['wait_time']), - query_time=float(job_dict['query_time']), error=job_dict['error'], - query_result_id=job_dict['query_result_id'], - process_id=job_dict['process_id']) - - return job - class Worker(threading.Thread): def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1): diff --git a/tests/test_job.py b/tests/test_job.py new file mode 100644 index 000000000..81f58d277 --- /dev/null +++ b/tests/test_job.py @@ -0,0 +1,90 @@ +import time +from unittest import TestCase +from mock import patch +from redash.data.worker import Job +from redash import redis_connection +from redash.utils import gen_query_hash + + +class TestJob(TestCase): + def setUp(self): + self.priority = 1 + self.query = "SELECT 1" + self.query_hash = gen_query_hash(self.query) + + def test_job_creation(self): + now = time.time() + with patch('time.time', return_value=now): + job = Job(redis_connection, query=self.query, priority=self.priority) + self.assertIsNotNone(job.id) + self.assertTrue(job.new_job) + self.assertEquals(0, job.wait_time) + self.assertEquals(0, job.query_time) + self.assertEquals(None, job.process_id) + self.assertEquals(Job.WAITING, job.status) + self.assertEquals(self.priority, job.priority) + self.assertEquals(self.query, job.query) + self.assertEquals(self.query_hash, job.query_hash) + self.assertIsNone(job.error) + self.assertIsNone(job.query_result_id) + + def test_job_loading(self): + job = Job(redis_connection, query=self.query, priority=self.priority) + job.save() + + loaded_job = Job.load(redis_connection, job.id) + + self.assertFalse(loaded_job.new_job) + + self.assertEquals(loaded_job.id, job.id) + self.assertEquals(loaded_job.wait_time, job.wait_time) + self.assertEquals(loaded_job.query_time, job.query_time) + self.assertEquals(loaded_job.process_id, job.process_id) + self.assertEquals(loaded_job.status, job.status) + self.assertEquals(loaded_job.priority, job.priority) + self.assertEquals(loaded_job.query_hash, job.query_hash) + self.assertEquals(loaded_job.query, job.query) + self.assertEquals(loaded_job.error, job.error) + self.assertEquals(loaded_job.query_result_id, job.query_result_id) + + def test_update(self): + job = Job(redis_connection, query=self.query, priority=self.priority) + + job.update(process_id=1) + self.assertEquals(1, job.process_id) + self.assertEquals(self.query, job.query) + self.assertEquals(self.priority, job.priority) + + def test_processing(self): + job = Job(redis_connection, query=self.query, priority=self.priority) + + updated_at = job.updated_at + now = time.time()+10 + with patch('time.time', return_value=now): + job.processing(10) + + job = Job.load(redis_connection, job.id) + + self.assertEquals(10, job.process_id) + self.assertEquals(Job.PROCESSING, job.status) + self.assertEquals(now, job.updated_at) + self.assertEquals(now - updated_at, job.wait_time) + + def test_done(self): + job = Job(redis_connection, query=self.query, priority=self.priority) + + updated_at = job.updated_at + now = time.time()+10 + with patch('time.time', return_value=now): + job.done(1, None) + + job = Job.load(redis_connection, job.id) + + self.assertEquals(Job.DONE, job.status) + self.assertEquals(1, job.query_result_id) + self.assertEquals(now, job.updated_at) + self.assertEquals(now - updated_at, job.query_time) + self.assertIsNone(job.error) + + def test_done_failed(self): + pass From 1d001407a0afac9270e2a9604570e4d10cd87a15 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 18 Mar 2014 20:45:03 +0200 Subject: [PATCH 386/540] Move query runner creation to worker based on data source in Job. --- redash/data/manager.py | 36 +++----------- redash/data/query_runner.py | 91 ++++++++++------------------------ redash/data/query_runner_pg.py | 68 +++++++++++++++++++++++++ redash/data/worker.py | 27 +++++++--- 4 files changed, 121 insertions(+), 101 deletions(-) create mode 100644 redash/data/query_runner_pg.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 08532d3b3..4cfd1e973 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -66,7 +66,7 @@ class Manager(object): return query_result - def add_job(self, query, priority): + def add_job(self, query, priority, data_source): query_hash = gen_query_hash(query) logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority) try_count = 0 @@ -83,7 +83,10 @@ class Manager(object): logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id) job = worker.Job.load(self.redis_connection, job_id) else: - job = worker.Job(self.redis_connection, query=query, priority=priority) + job = worker.Job(self.redis_connection, query=query, priority=priority, + data_source_name=data_source.name, + data_source_type=data_source.type, + data_source_options=data_source.options) pipe.multi() job.save(pipe) logging.info("[Manager][%s] Created new job: %s", query_hash, job.id) @@ -170,34 +173,9 @@ class Manager(object): if self.workers: return self.workers - if connection_type == 'mysql': - from redash.data import query_runner_mysql - runner = query_runner_mysql.mysql(connection_string) - elif connection_type == 'graphite': - from redash.data import query_runner_graphite - connection_params = json.loads(connection_string) - if connection_params['auth']: - connection_params['auth'] = tuple(connection_params['auth']) - else: - connection_params['auth'] = None - runner = query_runner_graphite.graphite(connection_params) - elif connection_type == 'bigquery': - from redash.data import query_runner_bigquery - connection_params = json.loads(connection_string) - runner = query_runner_bigquery.bigquery(connection_params) - elif connection_type == 'script': - from redash.data import query_runner_script - runner = query_runner_script.script(connection_string) - elif connection_type == 'url': - from redash.data import query_runner_url - runner = query_runner_url.url(connection_string) - else: - from redash.data import query_runner - runner = query_runner.redshift(connection_string) - redis_connection_params = self.redis_connection.connection_pool.connection_kwargs - self.workers = [worker.Worker(worker_id, self, redis_connection_params, runner) - for worker_id in range(workers_count)] + self.workers = [worker.Worker(worker_id, self, redis_connection_params) + for worker_id in xrange(workers_count)] for w in self.workers: w.start() diff --git a/redash/data/query_runner.py b/redash/data/query_runner.py index e44017e30..41e507952 100644 --- a/redash/data/query_runner.py +++ b/redash/data/query_runner.py @@ -1,69 +1,30 @@ -""" -QueryRunner is the function that the workers use, to execute queries. This is the Redshift -(PostgreSQL in fact) version, but easily we can write another to support additional databases -(MySQL and others). - -Because the worker just pass the query, this can be used with any data store that has some sort of -query language (for example: HiveQL). -""" -import logging import json -import sys -import select -import psycopg2 -from redash.utils import JSONEncoder +def get_query_runner(connection_type, connection_string): + if connection_type == 'mysql': + from redash.data import query_runner_mysql + runner = query_runner_mysql.mysql(connection_string) + elif connection_type == 'graphite': + from redash.data import query_runner_graphite + connection_params = json.loads(connection_string) + if connection_params['auth']: + connection_params['auth'] = tuple(connection_params['auth']) + else: + connection_params['auth'] = None + runner = query_runner_graphite.graphite(connection_params) + elif connection_type == 'bigquery': + from redash.data import query_runner_bigquery + connection_params = json.loads(connection_string) + runner = query_runner_bigquery.bigquery(connection_params) + elif connection_type == 'script': + from redash.data import query_runner_script + runner = query_runner_script.script(connection_string) + elif connection_type == 'url': + from redash.data import query_runner_url + runner = query_runner_url.url(connection_string) + else: + from redash.data import query_runner_pg + runner = query_runner_pg.pg(connection_string) -def redshift(connection_string): - def column_friendly_name(column_name): - return column_name - - def wait(conn): - while 1: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) - - def query_runner(query): - connection = psycopg2.connect(connection_string, async=True) - wait(connection) - - cursor = connection.cursor() - - try: - cursor.execute(query) - wait(connection) - - column_names = [col.name for col in cursor.description] - - rows = [dict(zip(column_names, row)) for row in cursor] - columns = [{'name': col.name, - 'friendly_name': column_friendly_name(col.name), - 'type': None} for col in cursor.description] - - data = {'columns': columns, 'rows': rows} - json_data = json.dumps(data, cls=JSONEncoder) - error = None - cursor.close() - except psycopg2.DatabaseError as e: - json_data = None - error = e.message - except KeyboardInterrupt: - connection.cancel() - error = "Query cancelled by user." - json_data = None - except Exception as e: - raise sys.exc_info()[1], None, sys.exc_info()[2] - finally: - connection.close() - - return json_data, error - - return query_runner + return runner \ No newline at end of file diff --git a/redash/data/query_runner_pg.py b/redash/data/query_runner_pg.py new file mode 100644 index 000000000..945de3872 --- /dev/null +++ b/redash/data/query_runner_pg.py @@ -0,0 +1,68 @@ +""" +QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL +version, but easily we can write another to support additional databases (MySQL and others). + +Because the worker just pass the query, this can be used with any data store that has some sort of +query language (for example: HiveQL). +""" +import json +import sys +import select + +import psycopg2 + +from redash.utils import JSONEncoder + + +def pg(connection_string): + def column_friendly_name(column_name): + return column_name + + def wait(conn): + while 1: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def query_runner(query): + connection = psycopg2.connect(connection_string, async=True) + wait(connection) + + cursor = connection.cursor() + + try: + cursor.execute(query) + wait(connection) + + column_names = [col.name for col in cursor.description] + + rows = [dict(zip(column_names, row)) for row in cursor] + columns = [{'name': col.name, + 'friendly_name': column_friendly_name(col.name), + 'type': None} for col in cursor.description] + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + cursor.close() + except psycopg2.DatabaseError as e: + json_data = None + error = e.message + except KeyboardInterrupt: + connection.cancel() + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + finally: + connection.close() + + return json_data, error + + return query_runner diff --git a/redash/data/worker.py b/redash/data/worker.py index 1c82d23de..e10ac8165 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -13,6 +13,7 @@ import setproctitle import redis from statsd import StatsClient from redash.utils import gen_query_hash +from redash.data.query_runner import get_query_runner from redash import settings @@ -99,7 +100,10 @@ class Job(RedisObject): 'updated_at': time.time, 'status': WAITING, 'process_id': None, - 'query_result_id': None + 'query_result_id': None, + 'data_source_name': None, + 'data_source_type': None, + 'data_source_options': None } conversions = { @@ -133,7 +137,9 @@ class Job(RedisObject): 'status': self.status, 'error': self.error, 'query_result_id': self.query_result_id, - 'process_id': self.process_id + 'process_id': self.process_id, + 'data_source_name': self.data_source_name, + 'data_source_type': self.data_source_type } def cancel(self): @@ -189,15 +195,15 @@ class Job(RedisObject): class Worker(threading.Thread): - def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1): + def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1): self.manager = manager self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() if k in ('host', 'db', 'password', 'port')} + self.continue_working = True - self.query_runner = query_runner self.sleep_time = sleep_time self.child_pid = None self.worker_id = worker_id @@ -282,15 +288,22 @@ class Worker(threading.Thread): start_time = time.time() self.set_title("running query %s" % job_id) - if getattr(self.query_runner, 'annotate_query', True): + logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id, + job.data_source_name, job.data_source_type) + + query_runner = get_query_runner(job.data_source_type, job.data_source_options) + + if getattr(query_runner, 'annotate_query', True): annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ (pid, job.id, job.query_hash, job.priority, job.query) else: annotated_query = job.query # TODO: here's the part that needs to be forked, not all of the worker process... - with self.statsd_client.timer('worker_{}.query_runner.run_time'.format(self.worker_id)): - data, error = self.query_runner(annotated_query) + with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id, + job.data_source_type, + job.data_source_name)): + data, error = query_runner(annotated_query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", From 2e078294c95042c352e169cdb2d0d50e1e3886df Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:19:43 +0200 Subject: [PATCH 387/540] Update angular-resource to 1.2.7 --- rd_ui/bower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 179b3176b..a6954a8a0 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -13,7 +13,7 @@ "angular-ui-codemirror": "0.0.5", "highcharts": "3.0.1", "underscore": "1.5.1", - "angular-resource": "1.0.7", + "angular-resource": "1.2.7", "angular-growl": "0.3.1", "angular-route": "1.2.7", "pivottable": "https://github.com/arikfr/pivottable.git", From b6dbc3356deb1d463278a4596d920bbb79efef3b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:22:15 +0200 Subject: [PATCH 388/540] dict representation for DataSource. --- redash/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/redash/models.py b/redash/models.py index cc0d99a17..18d11a6a5 100644 --- a/redash/models.py +++ b/redash/models.py @@ -114,6 +114,13 @@ class DataSource(BaseModel): class Meta: db_table = 'data_sources' + def to_dict(self): + return { + 'id': self.id, + 'name': self.name, + 'type': self.type + } + class Query(BaseModel): id = peewee.PrimaryKeyField() @@ -149,6 +156,7 @@ class Query(BaseModel): 'ttl': self.ttl, 'api_key': self.api_key, 'created_at': self.created_at, + 'data_source_id': self._data.get('data_source', None) } if with_user: From 8ae41c0b6a123fed5ac6ae0937949879f913d59f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:22:51 +0200 Subject: [PATCH 389/540] Show query's data source. --- rd_ui/app/scripts/controllers/query_view.js | 5 ++++- rd_ui/app/scripts/services/services.js | 17 ++++++++++++----- rd_ui/app/views/queryview.html | 6 ++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 63a737977..b2a0c41e5 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,12 +1,14 @@ (function () { 'use strict'; - var QueryViewCtrl = function ($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) { + var QueryViewCtrl = function ($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization, DataSource) { var DEFAULT_TAB = 'table'; var pristineHash = ""; var leavingPageText = "You will lose your changes if you leave"; var route = $route.current; + $scope.dataSources = DataSource.get(); + $scope.dirty = undefined; $scope.isNewQuery = false; $scope.isOwner = false; @@ -278,6 +280,7 @@ 'notifications', 'Query', 'Visualization', + 'DataSource', QueryViewCtrl ]); diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 8d95659a4..53cf156e1 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -284,10 +284,10 @@ return queryResult; } - QueryResult.get = function (query, ttl) { + QueryResult.get = function (data_source_id, query, ttl) { var queryResult = new QueryResult(); - QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) { + QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) { queryResult.update(response); if ('job' in response) { @@ -301,7 +301,7 @@ return QueryResult; }; - var Query = function ($resource, QueryResult) { + var Query = function ($resource, QueryResult, DataSource) { var Query = $resource('/api/queries/:id', {id: '@id'}); Query.prototype.getQueryResult = function(ttl) { @@ -316,7 +316,7 @@ } else if (this.latest_query_data_id && ttl != 0) { queryResult = QueryResult.getById(this.latest_query_data_id); } else { - queryResult = QueryResult.get(this.query, ttl); + queryResult = QueryResult.get(this.data_source_id, this.query, ttl); } return queryResult; @@ -329,7 +329,14 @@ return Query; }; + var DataSource = function($resource) { + var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}}); + + return DataSourceResource; + } + angular.module('redash.services', []) .factory('QueryResult', ['$resource', '$timeout', QueryResult]) - .factory('Query', ['$resource', 'QueryResult', Query]) + .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query]) + .factory('DataSource', ['$resource', DataSource]); })(); diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index 0e584b24b..db3bfd088 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -112,6 +112,12 @@

      +

      + + Data Source + +

      +

      From 6c9322624d4b61fe79312240ee46acfddce44414 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:23:38 +0200 Subject: [PATCH 390/540] Use datasource when executing queries. --- redash/controllers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/redash/controllers.py b/redash/controllers.py index 21e76edc4..13ce26169 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -129,6 +129,14 @@ class BaseResource(Resource): return current_user._get_current_object() +class DataSourceListAPI(BaseResource): + def get(self): + data_sources = [ds.to_dict() for ds in models.DataSource.select()] + return data_sources + +api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources') + + class DashboardListAPI(BaseResource): def get(self): dashboards = [d.to_dict() for d in @@ -328,7 +336,8 @@ class QueryResultListAPI(BaseResource): if query_result: return {'query_result': query_result.to_dict(parse_data=True)} else: - job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY) + data_source = models.DataSource.get_by_id(params['data_source_id']) + job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) return {'job': job.to_dict()} From dad207912e785b4ede0b55bc4283e319be6caee0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:34:26 +0200 Subject: [PATCH 391/540] Fix: query wasn't saving. --- redash/controllers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/redash/controllers.py b/redash/controllers.py index 13ce26169..30a75b8c9 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -242,6 +242,7 @@ class QueryListAPI(BaseResource): query_def.pop(field, None) query_def['user'] = self.current_user + query_def['data_source'] = query_def.pop('data_source_id') query = models.Query(**query_def) query.save() From d4e4afb97d911ad9ed3c6354cf365e5be0e414e6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:36:55 +0200 Subject: [PATCH 392/540] Put deprecation comment for data source settings. --- redash/controllers.py | 1 - redash/settings.py | 13 +------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 30a75b8c9..4b6b208e3 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -237,7 +237,6 @@ class QueryListAPI(BaseResource): @require_permission('create_query') def post(self): query_def = request.get_json(force=True) - # id, created_at, api_key for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']: query_def.pop(field, None) diff --git a/redash/settings.py b/redash/settings.py index a931085bf..6f437632b 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -46,19 +46,8 @@ STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash") NAME = os.environ.get('REDASH_NAME', 're:dash') -# "pg", "graphite", "mysql", "bigquery" or "script" +# The following is kept for backward compatability, and shouldn't be used any more. CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") -# Connection string for the database that is used to run queries against. Examples: -# -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" -# -- pg: CONNECTION_STRING = "user= password= host= port=5439 dbname=" -# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true} -# -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" } -# to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts -# -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS" (.i.e /home/user/redash_scripts/) -# all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING. -# The output of the scripts must be in the output format defined here: -# -- url: CONNECTION_STRING = "base URL" (i.e. http://myserver/somewhere) -# If CONNECTION_STRING is set, the query should be a relative URL. If it is not set a full URL can be used CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From 08b92e1f3d350cac5bc3fb4b4450c2d13c08fa1a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:39:08 +0200 Subject: [PATCH 393/540] Remove QueryFiddle ctrl. --- rd_ui/app/scripts/app.js | 5 - rd_ui/app/scripts/controllers/controllers.js | 215 ------------------- rd_ui/app/views/queryfiddle.html | 102 --------- 3 files changed, 322 deletions(-) delete mode 100644 rd_ui/app/views/queryfiddle.html diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index a3bf63bb3..2af66e403 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -62,11 +62,6 @@ angular.module('redash', [ 'query': ['Query', '$q', '$route', getQuery] } }); - $routeProvider.when('/queries/:queryId/fiddle', { - templateUrl: '/views/queryfiddle.html', - controller: 'QueryFiddleCtrl', - reloadOnSearch: false - }); $routeProvider.when('/queries/:queryId/source', { templateUrl: '/views/queryview.html', controller: 'QueryViewCtrl', diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index 5412b6611..3a8d14a1c 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -72,221 +72,6 @@ $scope.updateTime = ''; } - var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) { - var DEFAULT_TAB = 'table'; - var pristineHash = null; - var leavingPageText = "You will lose your changes if you leave"; - - $scope.dirty = undefined; - $scope.newVisualization = undefined; - - $window.onbeforeunload = function(){ - if (currentUser.canEdit($scope.query) && $scope.dirty) { - return leavingPageText; - } - } - - Mousetrap.bindGlobal("meta+s", function(e) { - e.preventDefault(); - - if (currentUser.canEdit($scope.query)) { - $scope.saveQuery(); - } - }); - - $scope.$on('$locationChangeStart', function(event, next, current) { - if (next.split("#")[0] == current.split("#")[0]) { - return; - } - - if (!currentUser.canEdit($scope.query)) { - return; - } - - if($scope.dirty && - !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { - event.preventDefault(); - } else { - Mousetrap.unbind("meta+s"); - } - }); - - $scope.$parent.pageTitle = "Query Fiddle"; - - $scope.$watch(function() {return $location.hash()}, function(hash) { - $scope.selectedTab = hash || DEFAULT_TAB; - }); - - $scope.lockButton = function (lock) { - $scope.queryExecuting = lock; - }; - - $scope.formatQuery = function() { - $scope.editorOptions.readOnly = 'nocursor'; - - $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) { - $scope.query.query = response; - $scope.editorOptions.readOnly = false; - }) - } - - $scope.saveQuery = function (duplicate, oldId) { - if (!oldId) { - oldId = $scope.query.id; - } - - delete $scope.query.latest_query_data; - $scope.query.$save(function (q) { - pristineHash = q.getHash(); - $scope.dirty = false; - - if (duplicate) { - growl.addInfoMessage("Query duplicated.", {ttl: 2000}); - } else{ - growl.addSuccessMessage("Query saved.", {ttl: 2000}); - } - - if (oldId != q.id) { - if (oldId == undefined) { - $location.path($location.path().replace('new', q.id)).replace(); - } else { - // TODO: replace this with a safer method - $location.path($location.path().replace(oldId, q.id)).replace(); - - // Reset visualizations tab to table after duplicating a query: - $location.hash('table'); - } - } - }, function(httpResponse) { - growl.addErrorMessage("Query could not be saved"); - }); - }; - - $scope.duplicateQuery = function () { - var oldId = $scope.query.id; - $scope.query.id = null; - $scope.query.ttl = -1; - - $scope.saveQuery(true, oldId); - }; - - // Query Editor: - $scope.editorOptions = { - mode: 'text/x-sql', - lineWrapping: true, - lineNumbers: true, - readOnly: false, - matchBrackets: true, - autoCloseBrackets: true - }; - - $scope.refreshOptions = [ - {value: -1, name: 'No Refresh'}, - {value: 60, name: 'Every minute'}, - ] - - _.each(_.range(1, 13), function(i) { - $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'}); - }) - - $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'}); - $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'}); - - $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) { - if (newError == undefined) { - return; - } - - if (oldError == undefined && newError != undefined) { - $scope.lockButton(false); - } - }); - - $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) { - if (!data) { - return; - } - - if ($scope.queryResult.getId() == null) { - $scope.dataUri = ""; - } else { - $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv'; - $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv"; - } - }); - - $scope.$watch("queryResult && queryResult.getStatus()", function (status) { - if (!status) { - return; - } - - if (status == "done") { - if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() && - $scope.query.query_hash == $scope.queryResult.query_result.query_hash) { - Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()}) - } - $scope.query.latest_query_data_id = $scope.queryResult.getId(); - - notifications.showNotification("re:dash", $scope.query.name + " updated."); - - $scope.lockButton(false); - } - }); - - if ($routeParams.queryId != undefined) { - $scope.query = Query.get({id: $routeParams.queryId}, function(q) { - pristineHash = q.getHash(); - $scope.dirty = false; - $scope.queryResult = $scope.query.getQueryResult(); - }); - } else { - $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser}); - $scope.lockButton(false); - } - - $scope.$watch('query.name', function() { - $scope.$parent.pageTitle = $scope.query.name; - }); - - $scope.$watch(function() { - return $scope.query.getHash(); - }, function(newHash) { - $scope.dirty = (newHash !== pristineHash); - }); - - $scope.executeQuery = function() { - $scope.queryResult = $scope.query.getQueryResult(0); - $scope.lockButton(true); - $scope.cancelling = false; - }; - - $scope.cancelExecution = function() { - $scope.cancelling = true; - $scope.queryResult.cancelExecution(); - }; - - $scope.deleteVisualization = function($e, vis) { - $e.preventDefault(); - if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { - Visualization.delete(vis); - if ($scope.selectedTab == vis.id) { - $scope.selectedTab = DEFAULT_TAB; - } - $scope.query.visualizations = - $scope.query.visualizations.filter(function(v) { - return vis.id !== v.id; - }); - } - }; - - unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { - if (newPanel && $routeParams.queryId == undefined) { - unbind(); - $scope.saveQuery(); - } - }); - } - var QueriesCtrl = function($scope, $http, $location, $filter, Query) { $scope.$parent.pageTitle = "All Queries"; $scope.gridConfig = { diff --git a/rd_ui/app/views/queryfiddle.html b/rd_ui/app/views/queryfiddle.html deleted file mode 100644 index 842864ac6..000000000 --- a/rd_ui/app/views/queryfiddle.html +++ /dev/null @@ -1,102 +0,0 @@ -

      -
      -
      -
      -

      -

      - -

      -

      -

      - -

      -
      -
      - - -
      - - Download Data Set - - - -
      - - - -
      -
      - -
      -
      - Query runtime: {{queryResult.getRuntime() | durationHumanize}} | - Last update time: | - Rows: {{queryResult.getData().length}} | - Created by: {{query.user.name}} -
      Refresh query:
      -
      -
      - -
      - Executing query... - -
      -
      - Query in queue... - -
      -
      Error running query: {{queryResult.getError()}}
      -
      - -
      -
        - - - - - × - - -
      - -
      - -
      - -
      - -
      - -
      -
      -

      -

      - -
      -

      -
      -
      -

      -

      - -
      -

      -
      -
      - -
      -
      -

      -

      - -
      -
      - -
      -

      -
      -
      -
      -
      \ No newline at end of file From f637ddf8ca3d53597d4726d682bff4eceef3ea60 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:42:36 +0200 Subject: [PATCH 394/540] Remove definition of QueryFIddleCtrl. --- rd_ui/app/scripts/controllers/controllers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index 3a8d14a1c..fb776e639 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -225,7 +225,6 @@ .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); From 259ea39d557ebd49182a674eeeb218d23e632f0e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:44:50 +0200 Subject: [PATCH 395/540] Move Highcharts color definitions to highchart's code file --- rd_ui/app/scripts/app.js | 5 +---- rd_ui/app/scripts/ng_highchart.js | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 2af66e403..b392aae63 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -85,9 +85,6 @@ angular.module('redash', [ redirectTo: '/' }); - Highcharts.setOptions({ - colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", - "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"] - }); + } ]); \ No newline at end of file diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index 09ef05280..a17de6802 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -1,6 +1,11 @@ (function () { 'use strict'; + Highcharts.setOptions({ + colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", + "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"] + }); + var defaultOptions = { title: { "text": null From 679921dc8e8f88902a8f17d0fb3684b90b8af776 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 11:45:38 +0200 Subject: [PATCH 396/540] Add DataSource to models list --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index 18d11a6a5..b7f8d2bfe 100644 --- a/redash/models.py +++ b/redash/models.py @@ -330,7 +330,7 @@ class Widget(BaseModel): def __unicode__(self): return u"%s" % self.id -all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog) +all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog) def create_db(create_tables, drop_tables): From 579ca28d6de3f1e6ffc3bf4794e499bf890fd469 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 12:53:30 +0200 Subject: [PATCH 397/540] Fix importer to use data source --- redash/import_export.py | 17 ++++++++++++++--- tests/test_import.py | 8 ++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/redash/import_export.py b/redash/import_export.py index 889850701..c5df47d9d 100644 --- a/redash/import_export.py +++ b/redash/import_export.py @@ -5,10 +5,11 @@ from flask.ext.script import Manager class Importer(object): - def __init__(self, object_mapping=None): + def __init__(self, object_mapping=None, data_source=None): if object_mapping is None: object_mapping = {} self.object_mapping = object_mapping + self.data_source = data_source def import_query_result(self, query_result): query_result = self._get_or_create(models.QueryResult, query_result['id'], @@ -30,7 +31,8 @@ class Importer(object): query=query['query'], query_hash=query['query_hash'], description=query['description'], - latest_query_data=query_result) + latest_query_data=query_result, + data_source=self.data_source) return new_query @@ -115,12 +117,21 @@ def importer_with_mapping_file(mapping_filename): with open(mapping_filename) as f: mapping = json.loads(f.read()) - importer = Importer(object_mapping=mapping) + importer = Importer(object_mapping=mapping, data_source=get_data_source()) yield importer with open(mapping_filename, 'w') as f: f.write(json.dumps(importer.object_mapping, indent=2)) + +def get_data_source(): + try: + data_source = models.DataSource.get(models.DataSource.name=="Import") + except models.DataSource.DoestNotExist: + data_source = models.DataSource.create(name="Import", type="import", options='{}') + + return data_source + @import_manager.command def query(mapping_filename, query_filename, user_id): user = models.User.get_by_id(user_id) diff --git a/tests/test_import.py b/tests/test_import.py index f4a6a5b27..a2fbabe7d 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -3,7 +3,7 @@ import os.path from tests import BaseTestCase from redash import models from redash import import_export -from factories import user_factory, dashboard_factory +from factories import user_factory, dashboard_factory, data_source_factory class ImportTest(BaseTestCase): @@ -15,7 +15,7 @@ class ImportTest(BaseTestCase): self.user = user_factory.create() def test_imports_dashboard_correctly(self): - importer = import_export.Importer() + importer = import_export.Importer(data_source=data_source_factory.create()) dashboard = importer.import_dashboard(self.user, self.dashboard) self.assertIsNotNone(dashboard) @@ -31,7 +31,7 @@ class ImportTest(BaseTestCase): self.assertEqual(models.QueryResult.select().count(), dashboard.widgets.count()-1) def test_imports_updates_existing_models(self): - importer = import_export.Importer() + importer = import_export.Importer(data_source=data_source_factory.create()) importer.import_dashboard(self.user, self.dashboard) self.dashboard['name'] = 'Testing #2' @@ -47,7 +47,7 @@ class ImportTest(BaseTestCase): } } - importer = import_export.Importer(object_mapping=mapping) + importer = import_export.Importer(object_mapping=mapping, data_source=data_source_factory.create()) imported_dashboard = importer.import_dashboard(self.user, self.dashboard) self.assertEqual(imported_dashboard, dashboard) \ No newline at end of file From 7eb849affb27b8a4d0b590d6a1f1b3d7e5672dc9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 12:53:39 +0200 Subject: [PATCH 398/540] Data Source factory --- tests/factories.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/factories.py b/tests/factories.py index be491d1ce..8bae0549d 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -43,6 +43,12 @@ user_factory = ModelFactory(redash.models.User, is_admin=False) +data_source_factory = ModelFactory(redash.models.DataSource, + name='Test', + type='pg', + options='') + + dashboard_factory = ModelFactory(redash.models.Dashboard, name='test', user=user_factory.create, layout='[]') @@ -52,7 +58,8 @@ query_factory = ModelFactory(redash.models.Query, description='', query='SELECT 1', ttl=-1, - user=user_factory.create) + user=user_factory.create, + data_source=data_source_factory.create) query_result_factory = ModelFactory(redash.models.QueryResult, data='{"columns":{}, "rows":[]}', @@ -73,4 +80,4 @@ widget_factory = ModelFactory(redash.models.Widget, width=1, options='{}', dashboard=dashboard_factory.create, - visualization=visualization_factory.create) + visualization=visualization_factory.create) \ No newline at end of file From 4fffcab8aa2d26082ee4423c92e81dec13e6defb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 12:53:51 +0200 Subject: [PATCH 399/540] Fix tests to use data source --- tests/test_controllers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 19aaeb876..759dfcc9e 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -7,7 +7,7 @@ from flask.ext.login import current_user from mock import patch from tests import BaseTestCase from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \ - query_result_factory, user_factory + query_result_factory, user_factory, data_source_factory from redash import app, models, settings from redash.utils import json_dumps from redash.authentication import sign @@ -211,10 +211,12 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin): def test_create_query(self): user = user_factory.create() + data_source = data_source_factory.create() query_data = { 'name': 'Testing', 'query': 'SELECT 1', - 'ttl': 3600 + 'ttl': 3600, + 'data_source_id': data_source.id } with app.test_client() as c, authenticated_user(c, user=user): From 0c199431a967347f1471631fc20a3a7e6d6add3d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 12:57:42 +0200 Subject: [PATCH 400/540] Add data source to QueryResult --- migrations/create_data_sources.py | 12 ++++++++++ redash/models.py | 37 ++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/migrations/create_data_sources.py b/migrations/create_data_sources.py index 52448a03f..45149117b 100644 --- a/migrations/create_data_sources.py +++ b/migrations/create_data_sources.py @@ -20,17 +20,29 @@ if __name__ == '__main__': migrator = Migrator(db.database) models.Query.data_source.null = True + models.QueryResult.data_source.null = True try: with db.database.transaction(): migrator.add_column(models.Query, models.Query.data_source, "data_source_id") except peewee.ProgrammingError: print "Failed to create data_source_id column -- assuming it already exists" + try: + with db.database.transaction(): + migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id") + except peewee.ProgrammingError: + print "Failed to create data_source_id column -- assuming it already exists" + print "Updating data source to existing one..." models.Query.update(data_source=default_data_source.id).execute() + models.QueryResult.update(data_source=default_data_source.id).execute() with db.database.transaction(): print "Setting data source to non nullable..." migrator.set_nullable(models.Query, models.Query.data_source, False) + with db.database.transaction(): + print "Setting data source to non nullable..." + migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False) + db.close_db(None) \ No newline at end of file diff --git a/redash/models.py b/redash/models.py index b7f8d2bfe..24fd02bf5 100644 --- a/redash/models.py +++ b/redash/models.py @@ -79,8 +79,27 @@ class ActivityLog(BaseModel): return unicode(self.id) +class DataSource(BaseModel): + id = peewee.PrimaryKeyField() + name = peewee.CharField() + type = peewee.CharField() + options = peewee.TextField() + created_at = peewee.DateTimeField(default=datetime.datetime.now) + + class Meta: + db_table = 'data_sources' + + def to_dict(self): + return { + 'id': self.id, + 'name': self.name, + 'type': self.type + } + + class QueryResult(BaseModel): id = peewee.PrimaryKeyField() + data_source = peewee.ForeignKeyField(DataSource) query_hash = peewee.CharField(max_length=32, index=True) query = peewee.TextField() data = peewee.TextField() @@ -104,24 +123,6 @@ class QueryResult(BaseModel): return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) -class DataSource(BaseModel): - id = peewee.PrimaryKeyField() - name = peewee.CharField() - type = peewee.CharField() - options = peewee.TextField() - created_at = peewee.DateTimeField(default=datetime.datetime.now) - - class Meta: - db_table = 'data_sources' - - def to_dict(self): - return { - 'id': self.id, - 'name': self.name, - 'type': self.type - } - - class Query(BaseModel): id = peewee.PrimaryKeyField() data_source = peewee.ForeignKeyField(DataSource) From 7f118635b4912c1c5de13253bd5616bf8b0f9519 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 13:47:35 +0200 Subject: [PATCH 401/540] Fix import job to use data source with query result. --- redash/import_export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/redash/import_export.py b/redash/import_export.py index c5df47d9d..51c0d9cde 100644 --- a/redash/import_export.py +++ b/redash/import_export.py @@ -13,6 +13,7 @@ class Importer(object): def import_query_result(self, query_result): query_result = self._get_or_create(models.QueryResult, query_result['id'], + data_source=self.data_source, data=json.dumps(query_result['data']), query_hash=query_result['query_hash'], retrieved_at=query_result['retrieved_at'], From 3f91ebea5f5163bf7349efcf1f9c7e15a03b1f55 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 13:48:11 +0200 Subject: [PATCH 402/540] Fix QueryResult factory. --- tests/factories.py | 6 ++++-- tests/test_controllers.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/factories.py b/tests/factories.py index 8bae0549d..73f591d36 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -1,5 +1,6 @@ import datetime import redash.models +from redash.utils import gen_query_hash class ModelFactory(object): @@ -65,8 +66,9 @@ query_result_factory = ModelFactory(redash.models.QueryResult, data='{"columns":{}, "rows":[]}', runtime=1, retrieved_at=datetime.datetime.now(), - query=query_factory.create, - query_hash='') + query="SELECT 1", + query_hash=gen_query_hash('SELECT 1'), + data_source=data_source_factory.create) visualization_factory = ModelFactory(redash.models.Visualization, type='CHART', diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 759dfcc9e..31412991e 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -306,12 +306,13 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): super(CsvQueryResultAPITest, self).setUp() self.paths = [] self.query_result = query_result_factory.create() - self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id) + self.query = query_factory.create() + self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id) # TODO: factor out the HMAC authentication tests def signature(self, expires): - return sign(self.query_result.query.api_key, self.path, expires) + return sign(self.query.api_key, self.path, expires) def test_redirect_when_unauthenticated(self): with app.test_client() as c: @@ -320,34 +321,34 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin): def test_redirect_for_wrong_signature(self): with app.test_client() as c: - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0}) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0}) self.assertEquals(rv.status_code, 302) def test_redirect_for_correct_signature_and_wrong_expires(self): with app.test_client() as c: - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0}) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0}) self.assertEquals(rv.status_code, 302) def test_redirect_for_correct_signature_and_no_expires(self): with app.test_client() as c: - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)}) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)}) self.assertEquals(rv.status_code, 302) def test_redirect_for_correct_signature_and_expires_too_long(self): with app.test_client() as c: expires = time.time()+(10*3600) - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) self.assertEquals(rv.status_code, 302) def test_returns_200_for_correct_signature(self): with app.test_client() as c: expires = time.time()+1800 - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires}) self.assertEquals(rv.status_code, 200) def test_returns_200_for_authenticated_user(self): with app.test_client() as c, authenticated_user(c): - rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)) + rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id)) self.assertEquals(rv.status_code, 200) From 193587dcfb1659feff539a6b755ab54038bcb036 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 13:48:48 +0200 Subject: [PATCH 403/540] Move QUeyrResult logic from data.Manager to QueryResult. --- redash/controllers.py | 2 +- redash/data/manager.py | 16 ------------ redash/models.py | 10 ++++++++ tests/test_models.py | 58 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 4b6b208e3..858168d68 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -331,7 +331,7 @@ class QueryResultListAPI(BaseResource): if params['ttl'] == 0: query_result = None else: - query_result = data_manager.get_query_result(params['query'], int(params['ttl'])) + query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict(parse_data=True)} diff --git a/redash/data/manager.py b/redash/data/manager.py index 4cfd1e973..546e5b688 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -50,22 +50,6 @@ class Manager(object): return query_result - def get_query_result(self, query, ttl=0): - query_hash = gen_query_hash(query) - - with self.db_transaction() as cursor: - sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \ - "WHERE query_hash=%s " \ - "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \ - "ORDER BY retrieved_at DESC LIMIT 1" - cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl))) - query_result = cursor.fetchone() - - if query_result: - query_result = QueryResult(*query_result) - - return query_result - def add_job(self, query, priority, data_source): query_hash = gen_query_hash(query) logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority) diff --git a/redash/models.py b/redash/models.py index 24fd02bf5..d2e57f6c5 100644 --- a/redash/models.py +++ b/redash/models.py @@ -115,10 +115,20 @@ class QueryResult(BaseModel): 'query_hash': self.query_hash, 'query': self.query, 'data': json.loads(self.data), + 'data_source_id': self._data.get('data_source', None), 'runtime': self.runtime, 'retrieved_at': self.retrieved_at } + @classmethod + def get_latest(cls, data_source, query, ttl=0): + query_hash = utils.gen_query_hash(query) + + query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source, + peewee.SQL("retrieved_at > now() at time zone 'utc' - interval '%s second'", ttl)).order_by(cls.retrieved_at.desc()) + + return query.first() + def __unicode__(self): return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at) diff --git a/tests/test_models.py b/tests/test_models.py index 171aba8b9..86c07f24e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,6 +1,7 @@ +import datetime from tests import BaseTestCase from redash import models -from factories import dashboard_factory, query_factory +from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory class DashboardTest(BaseTestCase): @@ -25,4 +26,57 @@ class QueryTest(BaseTestCase): q = models.Query.get_by_id(q.id) - self.assertNotEquals(old_hash, q.query_hash) \ No newline at end of file + self.assertNotEquals(old_hash, q.query_hash) + + +class QueryResultTest(BaseTestCase): + def setUp(self): + super(QueryResultTest, self).setUp() + + def test_get_latest_returns_none_if_not_found(self): + ds = data_source_factory.create() + models.QueryResult.get_latest(ds, "SELECT 1") + + def test_get_latest_returns_when_found(self): + qr = query_result_factory.create() + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query) + + self.assertEqual(qr, found_query_result) + + def test_get_latest_works_with_data_source_id(self): + qr = query_result_factory.create() + found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query) + + self.assertEqual(qr, found_query_result) + + def test_get_latest_doesnt_return_query_from_different_data_source(self): + qr = query_result_factory.create() + data_source = data_source_factory.create() + found_query_result = models.QueryResult.get_latest(data_source, qr.query) + + self.assertIsNone(found_query_result) + + def test_get_latest_doesnt_return_if_ttl_expired(self): + yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + qr = query_result_factory.create(retrieved_at=yesterday) + + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=60) + + self.assertIsNone(found_query_result) + + def test_get_latest_returns_if_ttl_not_expired(self): + yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30) + qr = query_result_factory.create(retrieved_at=yesterday) + + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=120) + + self.assertEqual(found_query_result, qr) + + def test_get_latest_returns_the_most_recent_result(self): + yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30) + old_qr = query_result_factory.create(retrieved_at=yesterday) + qr = query_result_factory.create() + + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query) + + self.assertEqual(found_query_result.id, qr.id) \ No newline at end of file From f186c8cb5f887b439bedfc3fecdaf4df826973aa Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 13:50:52 +0200 Subject: [PATCH 404/540] Remove get_query_result_by_id from data.Manager. --- redash/controllers.py | 4 ++-- redash/data/manager.py | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 858168d68..1ff8dd1fe 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -343,7 +343,7 @@ class QueryResultListAPI(BaseResource): class QueryResultAPI(BaseResource): def get(self, query_result_id): - query_result = data_manager.get_query_result_by_id(query_result_id) + query_result = models.QueryResult.get_by_id(query_result_id) if query_result: return {'query_result': query_result.to_dict(parse_data=True)} else: @@ -357,7 +357,7 @@ class CsvQueryResultsAPI(BaseResource): if query: query_result_id = query._data['latest_query_data'] - query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id) + query_result = query_result_id and models.QueryResult.get_by_id(query_result_id) if query_result: s = cStringIO.StringIO() diff --git a/redash/data/manager.py b/redash/data/manager.py index 546e5b688..fb8d2090e 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -37,19 +37,6 @@ class Manager(object): self._save_status() - # TODO: Use our Django Models - def get_query_result_by_id(self, query_result_id): - with self.db_transaction() as cursor: - sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \ - "WHERE id=%s LIMIT 1" - cursor.execute(sql, (query_result_id,)) - query_result = cursor.fetchone() - - if query_result: - query_result = QueryResult(*query_result) - - return query_result - def add_job(self, query, priority, data_source): query_hash = gen_query_hash(query) logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority) From 515eb28d4deb2b6300f9385c5a2c3d0d4623809b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 13:52:19 +0200 Subject: [PATCH 405/540] No need to pass connection string to workers --- manage.py | 2 +- redash/data/manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manage.py b/manage.py index 3f9ff11cf..db83c9b67 100755 --- a/manage.py +++ b/manage.py @@ -33,7 +33,7 @@ def runworkers(): logging.info("Cleaning old workers: %s", old_workers) - data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) + data_manager.start_workers(settings.WORKERS_COUNT) logging.info("Workers started.") while True: diff --git a/redash/data/manager.py b/redash/data/manager.py index fb8d2090e..78375e867 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -140,7 +140,7 @@ class Manager(object): return data - def start_workers(self, workers_count, connection_type, connection_string): + def start_workers(self, workers_count): if self.workers: return self.workers From ee3150fc6b11b138376674b2b89826ea853c521d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 19 Mar 2014 14:58:13 +0200 Subject: [PATCH 406/540] Update query results for same data source only --- redash/data/manager.py | 8 ++++---- redash/data/worker.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index 78375e867..d305606b7 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -6,7 +6,6 @@ import collections import json import time import logging -import psycopg2 import qr import redis from redash.data import worker @@ -55,6 +54,7 @@ class Manager(object): job = worker.Job.load(self.redis_connection, job_id) else: job = worker.Job(self.redis_connection, query=query, priority=priority, + data_source_id=data_source.id, data_source_name=data_source.name, data_source_type=data_source.type, data_source_options=data_source.options) @@ -108,7 +108,7 @@ class Manager(object): logging.info("Done refreshing queries... %d" % len(queries)) - def store_query_result(self, query, data, run_time, retrieved_at): + def store_query_result(self, data_source_id, query, data, run_time, retrieved_at): query_result_id = None query_hash = gen_query_hash(query) sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \ @@ -119,8 +119,8 @@ class Manager(object): query_result_id = cursor.fetchone()[0] logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id) - sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s" - cursor.execute(sql, (query_result_id, query_hash)) + sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s AND data_source_id=%s" + cursor.execute(sql, (query_result_id, query_hash, data_source_id)) logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount) else: diff --git a/redash/data/worker.py b/redash/data/worker.py index e10ac8165..ef63093a4 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -101,6 +101,7 @@ class Job(RedisObject): 'status': WAITING, 'process_id': None, 'query_result_id': None, + 'data_source_id': None, 'data_source_name': None, 'data_source_type': None, 'data_source_options': None @@ -314,7 +315,8 @@ class Worker(threading.Thread): query_result_id = None if not error: self.set_title("storing results %s" % job_id) - query_result_id = self.manager.store_query_result(job.query, data, run_time, + query_result_id = self.manager.store_query_result(job.data_source_id, + job.query, data, run_time, datetime.datetime.utcnow()) self.set_title("marking job as done %s" % job_id) From 8d8dafade3784079b07947983f74859385098c88 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 09:20:27 +0200 Subject: [PATCH 407/540] Allow updating data source when updating query. --- redash/controllers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 1ff8dd1fe..bd10c6d20 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -263,6 +263,9 @@ class QueryAPI(BaseResource): if 'latest_query_data_id' in query_def: query_def['latest_query_data'] = query_def.pop('latest_query_data_id') + if 'data_source_id' in query_def: + query_def['data_source'] = query_def.pop('data_source_id') + models.Query.update_instance(query_id, **query_def) query = models.Query.get_by_id(query_id) @@ -334,7 +337,7 @@ class QueryResultListAPI(BaseResource): query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl'])) if query_result: - return {'query_result': query_result.to_dict(parse_data=True)} + return {'query_result': query_result.to_dict()} else: data_source = models.DataSource.get_by_id(params['data_source_id']) job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) @@ -345,7 +348,7 @@ class QueryResultAPI(BaseResource): def get(self, query_result_id): query_result = models.QueryResult.get_by_id(query_result_id) if query_result: - return {'query_result': query_result.to_dict(parse_data=True)} + return {'query_result': query_result.to_dict()} else: abort(404) From 99b7e3126b019254e8d74418b25378d24cccca1a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 09:20:41 +0200 Subject: [PATCH 408/540] When updating query result, set the data source id. --- redash/data/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index d305606b7..96a7c7a10 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -111,10 +111,10 @@ class Manager(object): def store_query_result(self, data_source_id, query, data, run_time, retrieved_at): query_result_id = None query_hash = gen_query_hash(query) - sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \ - "VALUES (%s, %s, %s, %s, %s) RETURNING id" + sql = "INSERT INTO query_results (query_hash, query, data, runtime, data_source_id, retrieved_at) " \ + "VALUES (%s, %s, %s, %s, %s, %s) RETURNING id" with self.db_transaction() as cursor: - cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at)) + cursor.execute(sql, (query_hash, query, data, run_time, data_source_id, retrieved_at)) if cursor.rowcount == 1: query_result_id = cursor.fetchone()[0] logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id) From 872cee2228bcf82492c6993c8ee5e02f1c387969 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 09:21:01 +0200 Subject: [PATCH 409/540] Unless data source set already, set it to the first one. --- rd_ui/app/scripts/controllers/query_view.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index b2a0c41e5..09ed827b3 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -7,7 +7,9 @@ var leavingPageText = "You will lose your changes if you leave"; var route = $route.current; - $scope.dataSources = DataSource.get(); + $scope.dataSources = DataSource.get(function(dataSources) { + $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id; + }); $scope.dirty = undefined; $scope.isNewQuery = false; @@ -92,6 +94,7 @@ } delete $scope.query.latest_query_data; + $scope.query.$save(function (q) { pristineHash = q.getHash(); $scope.dirty = false; From 93f87f0922e5d1969e1ec96b31b9ac442af510d3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 09:29:27 +0200 Subject: [PATCH 410/540] Fix #141: prevent the timer directive call digest loop every second By using setInterval & $scope.$digest instead of $timeout which uses $scope.$apply, which in turn calls $rootScope.$digest. --- rd_ui/app/scripts/directives.js | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives.js index a41c9b863..da2b1f1de 100644 --- a/rd_ui/app/scripts/directives.js +++ b/rd_ui/app/scripts/directives.js @@ -302,31 +302,26 @@ }; }); - directives.directive('rdTimer', ['$timeout', function ($timeout) { + directives.directive('rdTimer', [function () { return { restrict: 'E', scope: { timestamp: '=' }, template: '{{currentTime}}', controller: ['$scope' ,function ($scope) { $scope.currentTime = "00:00:00"; - var currentTimeout = null; - var updateTime = function() { - $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss") - currentTimeout = $timeout(updateTime, 1000); - } - - var cancelTimer = function() { - if (currentTimeout) { - $timeout.cancel(currentTimeout); - currentTimeout = null; - } - } - - updateTime(); + // We're using setInterval directly instead of $timeout, to avoid using $apply, to + // prevent the digest loop being run every second. + var currentTimer = setInterval(function() { + $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss"); + $scope.$digest(); + }, 1000); $scope.$on('$destroy', function () { - cancelTimer(); + if (currentTimer) { + cancelInterval(currentTimer); + currentTimer = null; + } }); }] }; From f58ffd884b7e7e8dcd3d942d6a2627190b4b5544 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 10:47:07 +0200 Subject: [PATCH 411/540] Remove data.manager.QueryResult class. --- redash/data/manager.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/redash/data/manager.py b/redash/data/manager.py index 96a7c7a10..d0b8cf93c 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -11,15 +11,6 @@ import redis from redash.data import worker from redash.utils import gen_query_hash -class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')): - def to_dict(self, parse_data=False): - d = self._asdict() - - if parse_data and d['data']: - d['data'] = json.loads(d['data']) - - return d - class Manager(object): def __init__(self, redis_connection, db, statsd_client): From 2d3a0cc917bb92062eacc858c0eff32f9109716a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 13:18:32 +0200 Subject: [PATCH 412/540] Update peewee version to be able to use window functions. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5a2bb1cf..39b7e291c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ atfork==0.1.2 blinker==1.3 flask-peewee==0.6.5 itsdangerous==0.23 -peewee==2.2.0 +peewee==2.2.2 psycopg2==2.5.1 python-dateutil==2.1 pytz==2013.9 From 166b1a7c6be7af4bc3249e7f4777d17a0f83060d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 13:19:26 +0200 Subject: [PATCH 413/540] Switch to using peewee models in Manager + fix bugs + add tests (#8). --- redash/__init__.py | 2 +- redash/data/manager.py | 85 +++++++++-------------- tests/test_manager.py | 149 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 54 deletions(-) create mode 100644 tests/test_manager.py diff --git a/redash/__init__.py b/redash/__init__.py index 38c650202..9078b39b3 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -46,6 +46,6 @@ redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.por statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) from redash import data -data_manager = data.Manager(redis_connection, db, statsd_client) +data_manager = data.Manager(redis_connection, statsd_client) from redash import controllers diff --git a/redash/data/manager.py b/redash/data/manager.py index d0b8cf93c..62819820a 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -2,21 +2,20 @@ Data manager. Used to manage and coordinate execution of queries. """ from contextlib import contextmanager -import collections -import json import time import logging +import peewee import qr import redis +from redash import models from redash.data import worker from redash.utils import gen_query_hash class Manager(object): - def __init__(self, redis_connection, db, statsd_client): + def __init__(self, redis_connection, statsd_client): self.statsd_client = statsd_client self.redis_connection = redis_connection - self.db = db self.workers = [] self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 @@ -78,58 +77,53 @@ class Manager(object): time.time() - float(manager_status['last_refresh_at'])) def refresh_queries(self): - sql = """SELECT first_value(t1."query") over(partition by t1.query_hash) - FROM "queries" AS t1 - INNER JOIN "query_results" AS t2 ON (t1."latest_query_data_id" = t2."id") - WHERE ((t1."ttl" > 0) AND ((t2."retrieved_at" + t1.ttl * interval '1 second') < - now() at time zone 'utc')); - """ + # TODO: this will only execute scheduled queries that were executed before. I think this is + # a reasonable assumption, but worth revisiting. + outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\ + .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\ + .join(models.QueryResult)\ + .where(models.Query.ttl > 0, + (models.QueryResult.retrieved_at + + (models.Query.ttl * peewee.SQL("interval '1 second'"))) < + peewee.SQL("(now() at time zone 'utc')")) + + queries = models.Query.select(models.Query, models.DataSource).join(models.DataSource)\ + .where(models.Query.id << outdated_queries) self.status['last_refresh_at'] = time.time() self._save_status() logging.info("Refreshing queries...") - queries = self.run_query(sql) + outdated_queries_count = 0 for query in queries: - self.add_job(query[0], worker.Job.LOW_PRIORITY) + self.add_job(query.query, worker.Job.LOW_PRIORITY, query.data_source) + outdated_queries_count += 1 - self.statsd_client.gauge('manager.outdated_queries', len(queries)) + self.statsd_client.gauge('manager.outdated_queries', outdated_queries_count) self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs')) - logging.info("Done refreshing queries... %d" % len(queries)) + logging.info("Done refreshing queries... %d" % outdated_queries_count) def store_query_result(self, data_source_id, query, data, run_time, retrieved_at): - query_result_id = None query_hash = gen_query_hash(query) - sql = "INSERT INTO query_results (query_hash, query, data, runtime, data_source_id, retrieved_at) " \ - "VALUES (%s, %s, %s, %s, %s, %s) RETURNING id" - with self.db_transaction() as cursor: - cursor.execute(sql, (query_hash, query, data, run_time, data_source_id, retrieved_at)) - if cursor.rowcount == 1: - query_result_id = cursor.fetchone()[0] - logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id) - sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s AND data_source_id=%s" - cursor.execute(sql, (query_result_id, query_hash, data_source_id)) + query_result = models.QueryResult.create(query_hash=query_hash, + query=query, + runtime=run_time, + data_source=data_source_id, + retrieved_at=retrieved_at, + data=data) - logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount) - else: - logging.error("[Manager][%s] Failed inserting query data.", query_hash) - return query_result_id + logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id) - def run_query(self, *args): - sql = args[0] - logging.debug("running query: %s %s", sql, args[1:]) + updated_count = models.Query.update(latest_query_data=query_result).\ + where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\ + execute() - with self.db_transaction() as cursor: - cursor.execute(sql, args[1:]) - if cursor.description: - data = list(cursor) - else: - data = cursor.rowcount + logging.info("[Manager][%s] Updated %s queries.", query_hash, updated_count) - return data + return query_result.id def start_workers(self, workers_count): if self.workers: @@ -148,20 +142,5 @@ class Manager(object): w.continue_working = False w.join() - @contextmanager - def db_transaction(self): - self.db.connect_db() - - cursor = self.db.database.get_cursor() - try: - yield cursor - except: - self.db.database.rollback() - raise - else: - self.db.database.commit() - finally: - self.db.close_db(None) - def _save_status(self): self.redis_connection.hmset('manager:status', self.status) diff --git a/tests/test_manager.py b/tests/test_manager.py new file mode 100644 index 000000000..324b9b048 --- /dev/null +++ b/tests/test_manager.py @@ -0,0 +1,149 @@ +import datetime +from mock import patch, call +from tests import BaseTestCase +from redash.data import worker +from redash import data_manager, models +from tests.factories import query_factory, query_result_factory, data_source_factory +from redash.utils import gen_query_hash + + +class TestManagerRefresh(BaseTestCase): + def test_enqueues_outdated_queries(self): + query = query_factory.create(ttl=60) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + query.latest_query_data = query_result + query.save() + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source) + + def test_skips_fresh_queries(self): + query = query_factory.create(ttl=1200) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + self.assertFalse(add_job_mock.called) + + def test_skips_queries_with_no_ttl(self): + query = query_factory.create(ttl=-1) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + self.assertFalse(add_job_mock.called) + + def test_enqueues_query_only_once(self): + query = query_factory.create(ttl=60) + query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash, + data_source=query.data_source) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + query.latest_query_data = query_result + query2.latest_query_data = query_result + query.save() + query2.save() + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source) + + def test_enqueues_query_with_correct_data_source(self): + query = query_factory.create(ttl=60) + query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + query.latest_query_data = query_result + query2.latest_query_data = query_result + query.save() + query2.save() + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True) + self.assertEquals(2, add_job_mock.call_count) + + def test_enqueues_only_for_relevant_data_source(self): + query = query_factory.create(ttl=60) + query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash) + retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, + query_hash=query.query_hash) + query.latest_query_data = query_result + query2.latest_query_data = query_result + query.save() + query2.save() + + with patch('redash.data.Manager.add_job') as add_job_mock: + data_manager.refresh_queries() + add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source) + + +class TestManagerStoreResults(BaseTestCase): + def setUp(self): + super(TestManagerStoreResults, self).setUp() + self.data_source = data_source_factory.create() + self.query = "SELECT 1" + self.query_hash = gen_query_hash(self.query) + self.runtime = 123 + self.utcnow = datetime.datetime.utcnow() + self.data = "data" + + def test_stores_the_result(self): + query_result_id = data_manager.store_query_result(self.data_source.id, self.query, + self.data, self.runtime, self.utcnow) + + query_result = models.QueryResult.get_by_id(query_result_id) + + self.assertEqual(query_result.data, self.data) + self.assertEqual(query_result.runtime, self.runtime) + self.assertEqual(query_result.retrieved_at, self.utcnow) + self.assertEqual(query_result.query, self.query) + self.assertEqual(query_result.query_hash, self.query_hash) + self.assertEqual(query_result.data_source, self.data_source) + + def test_updates_existing_queries(self): + query1 = query_factory.create(query=self.query, data_source=self.data_source) + query2 = query_factory.create(query=self.query, data_source=self.data_source) + query3 = query_factory.create(query=self.query, data_source=self.data_source) + + query_result_id = data_manager.store_query_result(self.data_source.id, self.query, + self.data, self.runtime, self.utcnow) + + self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id) + self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id) + self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id) + + def test_doesnt_update_queries_with_different_hash(self): + query1 = query_factory.create(query=self.query, data_source=self.data_source) + query2 = query_factory.create(query=self.query, data_source=self.data_source) + query3 = query_factory.create(query=self.query + "123", data_source=self.data_source) + + query_result_id = data_manager.store_query_result(self.data_source.id, self.query, + self.data, self.runtime, self.utcnow) + + self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id) + self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id) + self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id) + + + def test_doesnt_update_queries_with_different_data_source(self): + query1 = query_factory.create(query=self.query, data_source=self.data_source) + query2 = query_factory.create(query=self.query, data_source=self.data_source) + query3 = query_factory.create(query=self.query, data_source=data_source_factory.create()) + + query_result_id = data_manager.store_query_result(self.data_source.id, self.query, + self.data, self.runtime, self.utcnow) + + self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id) + self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id) + self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id) \ No newline at end of file From 6989c7d2fd46bca6d67fa179cb662aae8e3c0dfc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 18:16:09 +0200 Subject: [PATCH 414/540] Script to test concurrency issues --- bin/test_multithreading.py | 52 ++++++++++++++++++++++++++++++++++++++ redash/data/manager.py | 1 - 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 bin/test_multithreading.py diff --git a/bin/test_multithreading.py b/bin/test_multithreading.py new file mode 100644 index 000000000..2cf30a534 --- /dev/null +++ b/bin/test_multithreading.py @@ -0,0 +1,52 @@ +""" +Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution. +""" +import atfork +atfork.monkeypatch_os_fork_functions() +import atfork.stdlib_fixer +atfork.stdlib_fixer.fix_logging_module() + +import time +from redash.data import worker +from redash import models, data_manager, redis_connection + +if __name__ == '__main__': + models.create_db(True, False) + + print "Creating data source..." + data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres") + + print "Clear jobs/hashes:" + redis_connection.delete("jobs") + query_hashes = redis_connection.keys("query_hash_*") + if query_hashes: + redis_connection.delete(*query_hashes) + + starting_query_results_count = models.QueryResult.select().count() + jobs_count = 5000 + workers_count = 10 + + print "Creating jobs..." + for i in xrange(jobs_count): + query = "SELECT {}".format(i) + print "Inserting: {}".format(query) + data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, + data_source=data_source) + + print "Starting workers..." + workers = data_manager.start_workers(workers_count) + + print "Waiting for jobs to be done..." + keep_waiting = True + while keep_waiting: + results_count = models.QueryResult.select().count() - starting_query_results_count + print "QueryResults: {}".format(results_count) + time.sleep(5) + if results_count == jobs_count: + print "Yay done..." + keep_waiting = False + + data_manager.stop_workers() + print "!!!TODO: verify results" + + print "Done." \ No newline at end of file diff --git a/redash/data/manager.py b/redash/data/manager.py index 62819820a..a3b129dd0 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -1,7 +1,6 @@ """ Data manager. Used to manage and coordinate execution of queries. """ -from contextlib import contextmanager import time import logging import peewee From addaf974899c245b69e095f280d340b234229770 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 18:49:38 +0200 Subject: [PATCH 415/540] Add results verification --- bin/test_multithreading.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bin/test_multithreading.py b/bin/test_multithreading.py index 2cf30a534..5eae91623 100644 --- a/bin/test_multithreading.py +++ b/bin/test_multithreading.py @@ -1,6 +1,7 @@ """ Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution. """ +import json import atfork atfork.monkeypatch_os_fork_functions() import atfork.stdlib_fixer @@ -47,6 +48,16 @@ if __name__ == '__main__': keep_waiting = False data_manager.stop_workers() - print "!!!TODO: verify results" + + qr_count = 0 + for qr in models.QueryResult.select(): + number = int(qr.query.split()[1]) + data_number = json.loads(qr.data)['rows'][0].values()[0] + + if number != data_number: + print "Oops? {} != {} ({})".format(number, data_number, qr.id) + qr_count += 1 + + print "Verified {} query results.".format(qr_count) print "Done." \ No newline at end of file From 366cdbf616c099bb878949cde60b78d19019b5ac Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 19:30:05 +0200 Subject: [PATCH 416/540] Remove reference to query result when changing data source --- rd_ui/app/scripts/controllers/query_view.js | 13 +++++++++++++ rd_ui/app/scripts/services/services.js | 1 - rd_ui/app/views/queryview.html | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 09ed827b3..7b93a3f98 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -238,6 +238,19 @@ $scope.dirty = (newHash !== pristineHash); }); + $scope.updateDataSource = function() { + $scope.query.latest_query_data = null; + $scope.query.latest_query_data_id = null; + Query.save({ + 'id': $scope.query.id, + 'data_source_id': $scope.query.data_source_id, + 'latest_query_data_id': null + }); + + $scope.executeQuery(); + + }; + $scope.executeQuery = function () { $scope.queryResult = $scope.query.getQueryResult(0); $scope.lockButton(true); diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 53cf156e1..225295c2c 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -309,7 +309,6 @@ ttl = this.ttl; } - var queryResult = null; if (this.latest_query_data && ttl != 0) { queryResult = new QueryResult({'query_result': this.latest_query_data}); diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index db3bfd088..38e3dff33 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -115,7 +115,7 @@

      Data Source - +

      From 86b95a404a275296bb996b7de8e8d5072db10ff3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 19:38:05 +0200 Subject: [PATCH 417/540] Apply filters only when available --- rd_ui/app/scripts/visualizations/base.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 5a0a34daf..c60a1573a 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -67,7 +67,11 @@ template: '\n' + Visualization.renderVisualizationsTemplate, replace: false, link: function(scope) { - scope.filters = scope.queryResult.getFilters(); + scope.$watch('queryResult && queryResult.getFilters()', function(filters) { + if (filters) { + scope.filters = filters; + } + }); } } }; From 5979d91875e61d04523c5cbfca29d4cce86f2111 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 19:45:13 +0200 Subject: [PATCH 418/540] Reduce Peewee's logging level to INFO in tests. --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index f29714713..8fc307505 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,4 @@ +import logging from unittest import TestCase from redash import settings, db, app import redash.models @@ -11,6 +12,8 @@ settings.DATABASE_CONFIG = { app.config['DATABASE'] = settings.DATABASE_CONFIG db.load_database() +logging.getLogger('peewee').setLevel(logging.INFO) + for model in redash.models.all_models: model._meta.database = db.database From 725a8f2bb5f3fcb716c61732872b3b8cf80ac25b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 20:16:23 +0200 Subject: [PATCH 419/540] Reverse comparison --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index d2e57f6c5..3bc041c36 100644 --- a/redash/models.py +++ b/redash/models.py @@ -125,7 +125,7 @@ class QueryResult(BaseModel): query_hash = utils.gen_query_hash(query) query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source, - peewee.SQL("retrieved_at > now() at time zone 'utc' - interval '%s second'", ttl)).order_by(cls.retrieved_at.desc()) + peewee.SQL("retrieved_at + interval '%s second' > now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc()) return query.first() From d7a453e8b1745751ed841350ceee015761f41443 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 20 Mar 2014 20:57:35 +0200 Subject: [PATCH 420/540] Fix tests (were rightfully failing on system with clock set to utc) --- redash/models.py | 2 +- tests/factories.py | 2 +- tests/test_models.py | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/redash/models.py b/redash/models.py index 3bc041c36..8fd484b57 100644 --- a/redash/models.py +++ b/redash/models.py @@ -125,7 +125,7 @@ class QueryResult(BaseModel): query_hash = utils.gen_query_hash(query) query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source, - peewee.SQL("retrieved_at + interval '%s second' > now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc()) + peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc()) return query.first() diff --git a/tests/factories.py b/tests/factories.py index 73f591d36..7327c5189 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -65,7 +65,7 @@ query_factory = ModelFactory(redash.models.Query, query_result_factory = ModelFactory(redash.models.QueryResult, data='{"columns":{}, "rows":[]}', runtime=1, - retrieved_at=datetime.datetime.now(), + retrieved_at=datetime.datetime.utcnow, query="SELECT 1", query_hash=gen_query_hash('SELECT 1'), data_source=data_source_factory.create) diff --git a/tests/test_models.py b/tests/test_models.py index 86c07f24e..854929c5b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -35,24 +35,25 @@ class QueryResultTest(BaseTestCase): def test_get_latest_returns_none_if_not_found(self): ds = data_source_factory.create() - models.QueryResult.get_latest(ds, "SELECT 1") + found_query_result = models.QueryResult.get_latest(ds, "SELECT 1", 60) + self.assertIsNone(found_query_result) def test_get_latest_returns_when_found(self): qr = query_result_factory.create() - found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query) + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60) self.assertEqual(qr, found_query_result) def test_get_latest_works_with_data_source_id(self): qr = query_result_factory.create() - found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query) + found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query, 60) self.assertEqual(qr, found_query_result) def test_get_latest_doesnt_return_query_from_different_data_source(self): qr = query_result_factory.create() data_source = data_source_factory.create() - found_query_result = models.QueryResult.get_latest(data_source, qr.query) + found_query_result = models.QueryResult.get_latest(data_source, qr.query, 60) self.assertIsNone(found_query_result) @@ -77,6 +78,6 @@ class QueryResultTest(BaseTestCase): old_qr = query_result_factory.create(retrieved_at=yesterday) qr = query_result_factory.create() - found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query) + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60) self.assertEqual(found_query_result.id, qr.id) \ No newline at end of file From af85943c08af4aeea975ffddf07d852e5aa0b9f3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 23 Mar 2014 12:52:22 +0200 Subject: [PATCH 421/540] Add comment about moving logic to the model --- redash/data/manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/redash/data/manager.py b/redash/data/manager.py index a3b129dd0..c5edcdb82 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -78,6 +78,8 @@ class Manager(object): def refresh_queries(self): # TODO: this will only execute scheduled queries that were executed before. I think this is # a reasonable assumption, but worth revisiting. + + # TODO: move this logic to the model. outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\ .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\ .join(models.QueryResult)\ @@ -116,6 +118,7 @@ class Manager(object): logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id) + # TODO: move this logic to the model? updated_count = models.Query.update(latest_query_data=query_result).\ where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\ execute() From b8dba487596b32eb112505f179ef1c3f275b06af Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 24 Mar 2014 14:18:03 +0200 Subject: [PATCH 422/540] Fix issue with serializing unicode queries --- redash/data/worker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/redash/data/worker.py b/redash/data/worker.py index ef63093a4..113325929 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -80,6 +80,13 @@ class RedisObject(object): return obj +def fix_unicode(string): + if isinstance(string, unicode): + return string + + return string.decode('utf-8') + + class Job(RedisObject): HIGH_PRIORITY = 1 LOW_PRIORITY = 2 @@ -108,7 +115,7 @@ class Job(RedisObject): } conversions = { - 'query': lambda query: query.decode('utf-8'), + 'query': fix_unicode, 'priority': int, 'updated_at': float, 'status': int, @@ -121,7 +128,7 @@ class Job(RedisObject): name = 'job' def __init__(self, redis_connection, query, priority, **kwargs): - kwargs['query'] = query + kwargs['query'] = fix_unicode(query) kwargs['priority'] = priority kwargs['query_hash'] = gen_query_hash(kwargs['query']) self.new_job = 'id' not in kwargs From a1f81705dd337d1c5ebfcd1afc2937d9b13135dc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 24 Mar 2014 14:26:57 +0200 Subject: [PATCH 423/540] Unicode test case for Job --- tests/test_job.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 81f58d277..68efa013e 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import time from unittest import TestCase from mock import patch @@ -86,5 +88,13 @@ class TestJob(TestCase): self.assertEquals(now - updated_at, job.query_time) self.assertIsNone(job.error) - def test_done_failed(self): - pass + def test_unicode_serialization(self): + unicode_query = u"יוניקוד" + job = Job(redis_connection, query=unicode_query, priority=self.priority) + + self.assertEquals(job.query, unicode_query) + + job.save() + loaded_job = Job.load(redis_connection, job.id) + self.assertEquals(loaded_job.query, unicode_query) + From 37d56a2bf649383eb7b06b2b19b174dc20025a2b Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 13 Mar 2014 16:25:23 +0200 Subject: [PATCH 424/540] [#138] editCtrl inheriting viewCtrl --- rd_ui/app/index.html | 1 + rd_ui/app/scripts/app.js | 28 +-- rd_ui/app/scripts/controllers/query_edit.js | 122 ++++++++++++ rd_ui/app/scripts/controllers/query_view.js | 202 +++----------------- rd_ui/app/views/query.html | 171 +++++++++++++++++ rd_ui/app/views/queryview.html | 7 +- 6 files changed, 333 insertions(+), 198 deletions(-) create mode 100644 rd_ui/app/scripts/controllers/query_edit.js create mode 100644 rd_ui/app/views/query.html diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 4441e0ae9..e6c1b096d 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -121,6 +121,7 @@ + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index b392aae63..1b7006523 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -42,35 +42,25 @@ angular.module('redash', [ controller: 'QueriesCtrl', reloadOnSearch: false }); - $routeProvider.when('/queries/new', { - templateUrl: '/views/queryview.html', - controller: 'QueryViewCtrl', - reloadOnSearch: false, - resolve: { - 'viewSource': function isViewSource() { - return true; - } - } - }); - // TODO - // we should have 2 controllers: queryViewCtrl and queryEditCtrl $routeProvider.when('/queries/:queryId', { - templateUrl: '/views/queryview.html', + templateUrl: '/views/query.html', controller: 'QueryViewCtrl', reloadOnSearch: false, resolve: { 'query': ['Query', '$q', '$route', getQuery] } }); + $routeProvider.when('/queries/new', { + templateUrl: '/views/query.html', + controller: 'QueryEditCtrl', + reloadOnSearch: false + }); $routeProvider.when('/queries/:queryId/source', { - templateUrl: '/views/queryview.html', - controller: 'QueryViewCtrl', + templateUrl: '/views/query.html', + controller: 'QueryEditCtrl', reloadOnSearch: false, resolve: { - 'query': ['Query', '$q', '$route', getQuery], - 'viewSource': function isViewSource() { - return true; - } + 'query': ['Query', '$q', '$route', getQuery] } }); $routeProvider.when('/admin/status', { diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js new file mode 100644 index 000000000..00a59bca4 --- /dev/null +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -0,0 +1,122 @@ +(function () { + 'use strict'; + + function QueryEditCtrl($controller, $scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) { + var pristineHash = ""; + var leavingPageText = "You will lose your changes if you leave"; + + $controller('QueryViewCtrl', {$scope: $scope}); + + $scope.dirty = undefined; + $scope.isNewQuery = false; + + $scope.isSourceVisible = true; + + $scope.newVisualization = undefined; + + $window.onbeforeunload = function () { + if ($scope.canEdit && $scope.dirty) { + return leavingPageText; + } + } + + function getQuerySourceUrl(queryId) { + return '/queries/' + queryId + '/source#' + $location.hash(); + }; + + Mousetrap.bindGlobal("meta+s", function (e) { + e.preventDefault(); + + if ($scope.canEdit) { + $scope.saveQuery(); + } + }); + + $scope.$on('$locationChangeStart', function (event, next, current) { + if (next.split("#")[0] == current.split("#")[0]) { + return; + } + + if (!$scope.canEdit) { + return; + } + + if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { + event.preventDefault(); + } else { + Mousetrap.unbind("meta+s"); + } + }); + + $scope.saveQuery = function (duplicate, oldId) { + if (!oldId) { + oldId = $scope.query.id; + } + + delete $scope.query.latest_query_data; + $scope.query.$save(function (q) { + pristineHash = q.getHash(); + $scope.dirty = false; + + if (duplicate) { + growl.addSuccessMessage("Query forked"); + } else { + growl.addSuccessMessage("Query saved"); + } + + if (oldId != q.id) { + $location.url(getQuerySourceUrl(q.id)).replace(); + } + }, function (httpResponse) { + growl.addErrorMessage("Query could not be saved"); + }); + }; + + + // new query + if (!$route.current.locals.query) { + $scope.query = new Query({ + query: "", + name: "New Query", + ttl: -1, + user: currentUser + }); + $scope.lockButton(false); + $scope.isOwner = $scope.canEdit = true; + $scope.isNewQuery = true; + + var unbind = $scope.$watch('selectedTab == "add"', function (newPanel) { + if (newPanel && route.params.queryId == undefined) { + unbind(); + $scope.saveQuery(); + } + }); + } + + $scope.$watch(function () { + return $scope.query.getHash(); + }, function (newHash) { + $scope.dirty = (newHash !== pristineHash); + }); + + + $scope.deleteVisualization = function ($e, vis) { + $e.preventDefault(); + if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { + Visualization.delete(vis); + if ($scope.selectedTab == vis.id) { + $scope.selectedTab = DEFAULT_TAB; + $location.hash($scope.selectedTab); + } + $scope.query.visualizations = + $scope.query.visualizations.filter(function (v) { + return vis.id !== v.id; + }); + } + }; + }; + + angular.module('redash.controllers') + .controller('QueryEditCtrl', ['$controller', '$scope', '$window', '$route', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryEditCtrl]); + +})(); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 7b93a3f98..51e47d969 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,78 +1,24 @@ (function () { 'use strict'; - var QueryViewCtrl = function ($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization, DataSource) { + function QueryViewCtrl($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) { var DEFAULT_TAB = 'table'; - var pristineHash = ""; - var leavingPageText = "You will lose your changes if you leave"; var route = $route.current; - $scope.dataSources = DataSource.get(function(dataSources) { - $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id; - }); - - $scope.dirty = undefined; - $scope.isNewQuery = false; - $scope.isOwner = false; - $scope.canEdit = false; - $scope.canFork = false; - - $scope.isSourceVisible = route.locals.viewSource; - + $scope.query = route.locals.query; + $scope.queryResult = $scope.query.getQueryResult(); $scope.queryExecuting = false; - $scope.newVisualization = undefined; + $scope.isOwner = currentUser.canEdit($scope.query); + $scope.canEdit = $scope.isSourceVisible && $scope.isOwner; + $scope.canFork = true; - updateSourceHref(); - - $window.onbeforeunload = function () { - if ($scope.canEdit && $scope.dirty) { - return leavingPageText; - } - } - - function updateSourceHref() { - if ($scope.query && $scope.query.id) { - $scope.sourceHref = $scope.isSourceVisible ? - $location.url().replace('source', '') : getQuerySourceUrl($scope.query.id); - } - }; + $scope.sourceHref = getQuerySourceUrl(); function getQuerySourceUrl(queryId) { - return '/queries/' + queryId + '/source#' + $location.hash(); + return '/queries/' + $scope.query.id + '/source#' + $location.hash(); }; - Mousetrap.bindGlobal("meta+s", function (e) { - e.preventDefault(); - - if ($scope.canEdit) { - $scope.saveQuery(); - } - }); - - $scope.$on('$locationChangeStart', function (event, next, current) { - if (next.split("#")[0] == current.split("#")[0]) { - return; - } - - if (!$scope.canEdit) { - return; - } - - if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { - event.preventDefault(); - } else { - Mousetrap.unbind("meta+s"); - } - }); - - $scope.$watch(function () { - return $location.hash() - }, function (hash) { - $scope.selectedTab = hash || DEFAULT_TAB; - updateSourceHref(); - }); - $scope.lockButton = function (lock) { $scope.queryExecuting = lock; }; @@ -88,31 +34,6 @@ }) } - $scope.saveQuery = function (duplicate, oldId) { - if (!oldId) { - oldId = $scope.query.id; - } - - delete $scope.query.latest_query_data; - - $scope.query.$save(function (q) { - pristineHash = q.getHash(); - $scope.dirty = false; - - if (duplicate) { - growl.addSuccessMessage("Query forked"); - } else { - growl.addSuccessMessage("Query saved"); - } - - if (oldId != q.id) { - $location.url(getQuerySourceUrl(q.id)).replace(); - } - }, function (httpResponse) { - growl.addErrorMessage("Query could not be saved"); - }); - }; - $scope.duplicateQuery = function () { var oldId = $scope.query.id; $scope.query.id = null; @@ -121,7 +42,17 @@ $scope.saveQuery(true, oldId); }; - // Query Editor: + $scope.executeQuery = function () { + $scope.queryResult = $scope.query.getQueryResult(0); + $scope.lockButton(true); + $scope.cancelling = false; + }; + + $scope.cancelExecution = function () { + $scope.cancelling = true; + $scope.queryResult.cancelExecution(); + }; + $scope.editorOptions = { mode: 'text/x-sql', lineWrapping: true, @@ -204,100 +135,17 @@ } }); - // view or source pages: controller is instantiated with a query - if (route.locals.query) { - $scope.query = route.locals.query; - pristineHash = $scope.query.getHash(); - $scope.dirty = false; - $scope.queryResult = $scope.query.getQueryResult(); - - $scope.isOwner = currentUser.canEdit($scope.query); - $scope.canEdit = $scope.isSourceVisible && $scope.isOwner; - $scope.canFork = true; - - } else { - // new query - $scope.query = new Query({ - query: "", - name: "New Query", - ttl: -1, - user: currentUser - }); - $scope.lockButton(false); - $scope.isOwner = $scope.canEdit = true; - $scope.isNewQuery = true; - } - $scope.$watch('query.name', function () { $scope.$parent.pageTitle = $scope.query.name; }); - $scope.$watch(function () { - return $scope.query.getHash(); - }, function (newHash) { - $scope.dirty = (newHash !== pristineHash); - }); - - $scope.updateDataSource = function() { - $scope.query.latest_query_data = null; - $scope.query.latest_query_data_id = null; - Query.save({ - 'id': $scope.query.id, - 'data_source_id': $scope.query.data_source_id, - 'latest_query_data_id': null - }); - - $scope.executeQuery(); - - }; - - $scope.executeQuery = function () { - $scope.queryResult = $scope.query.getQueryResult(0); - $scope.lockButton(true); - $scope.cancelling = false; - }; - - $scope.cancelExecution = function () { - $scope.cancelling = true; - $scope.queryResult.cancelExecution(); - }; - - $scope.deleteVisualization = function ($e, vis) { - $e.preventDefault(); - if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { - Visualization.delete(vis); - if ($scope.selectedTab == vis.id) { - $scope.selectedTab = DEFAULT_TAB; - $location.hash($scope.selectedTab); - } - $scope.query.visualizations = - $scope.query.visualizations.filter(function (v) { - return vis.id !== v.id; - }); - } - }; - - var unbind = $scope.$watch('selectedTab == "add"', function (newPanel) { - if (newPanel && route.params.queryId == undefined) { - unbind(); - $scope.saveQuery(); - } + return $location.hash() + }, function (hash) { + $scope.selectedTab = hash || DEFAULT_TAB; + $scope.sourceHref = getQuerySourceUrl(); }); }; - angular.module('redash.controllers').controller('QueryViewCtrl', - [ - '$scope', - '$window', - '$route', - '$http', - '$location', - 'growl', - 'notifications', - 'Query', - 'Visualization', - 'DataSource', - QueryViewCtrl - ]); - + angular.module('redash.controllers') + .controller('QueryViewCtrl', ['$scope', '$window', '$route', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryViewCtrl]); })(); \ No newline at end of file diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html new file mode 100644 index 000000000..d8d4df811 --- /dev/null +++ b/rd_ui/app/views/query.html @@ -0,0 +1,171 @@ + +
      +
      +
      +
      +
      +

      + +

      +

      + + + +

      +
      +
      + +
      +
      + + + +
      +

      + Last update + + + +   + Created By + {{query.user.name}} + You +   + Runtime + {{queryResult.getRuntime() | durationHumanize}} + Running… +   + Rows + {{queryResult.getData().length}} +

      +

      + + Hide Source + View Source + +

      +
      +
      +
      + +
      + +
      +
      +
      +

      + + + + + + + +

      +
      + + +

      + +

      +
      +
      + +
      + +
      +
      +

      + + Last update + + + +

      +

      + + Created By + {{query.user.name}} + You +

      +

      + + Runtime + {{queryResult.getRuntime() | durationHumanize}} + Running… +

      +

      + + Rows {{queryResult.getData().length}} +

      +

      + + Refresh Interval + +

      + +
      + +

      + + + Download Dataset + +

      +
      + +
      + +
      + Executing query… + +
      +
      + Query in queue… + +
      +
      Error running query: {{queryResult.getError()}}
      + + +
      +
      +
      +
        + + + + × + + +
      +
      +
      +
      +
      + + + +
      + + +
      + +
      + + +
      +
      +
      +
      +
      +
      +
      diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html index 38e3dff33..3bb5c54fb 100644 --- a/rd_ui/app/views/queryview.html +++ b/rd_ui/app/views/queryview.html @@ -31,7 +31,8 @@   Created By - {{query.user.name}} + {{query.user.name}} + You   Runtime {{queryResult.getRuntime() | durationHumanize}} @@ -94,7 +95,9 @@

      - Created By {{query.user.name}} + Created By + {{query.user.name}} + You

      From 3ad8114a2833dfc1e63e1f56f4159cf24a72f863 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 19 Mar 2014 10:11:31 +0200 Subject: [PATCH 425/540] [#138] query directives --- rd_ui/app/index.html | 3 +- rd_ui/app/scripts/controllers/query_view.js | 45 ----- .../scripts/{ => directives}/directives.js | 0 .../scripts/directives/query_directives.js | 102 ++++++++++ rd_ui/app/views/query.html | 4 +- rd_ui/app/views/queryview.html | 183 ------------------ 6 files changed, 106 insertions(+), 231 deletions(-) rename rd_ui/app/scripts/{ => directives}/directives.js (100%) create mode 100644 rd_ui/app/scripts/directives/query_directives.js delete mode 100644 rd_ui/app/views/queryview.html diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index e6c1b096d..64e476d3f 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -127,7 +127,8 @@ - + + diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 51e47d969..801873b31 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -23,16 +23,7 @@ $scope.queryExecuting = lock; }; - $scope.formatQuery = function () { - $scope.editorOptions.readOnly = 'nocursor'; - $http.post('/api/queries/format', { - 'query': $scope.query.query - }).success(function (response) { - $scope.query.query = response; - $scope.editorOptions.readOnly = false; - }) - } $scope.duplicateQuery = function () { var oldId = $scope.query.id; @@ -53,42 +44,6 @@ $scope.queryResult.cancelExecution(); }; - $scope.editorOptions = { - mode: 'text/x-sql', - lineWrapping: true, - lineNumbers: true, - readOnly: false, - matchBrackets: true, - autoCloseBrackets: true - }; - - $scope.refreshOptions = [ - { - value: -1, - name: 'No Refresh' - }, - { - value: 60, - name: 'Every minute' - }, - ] - - _.each(_.range(1, 13), function (i) { - $scope.refreshOptions.push({ - value: i * 3600, - name: 'Every ' + i + 'h' - }); - }) - - $scope.refreshOptions.push({ - value: 24 * 3600, - name: 'Every 24h' - }); - $scope.refreshOptions.push({ - value: 7 * 24 * 3600, - name: 'Once a week' - }); - $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) { if (newError == undefined) { return; diff --git a/rd_ui/app/scripts/directives.js b/rd_ui/app/scripts/directives/directives.js similarity index 100% rename from rd_ui/app/scripts/directives.js rename to rd_ui/app/scripts/directives/directives.js diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js new file mode 100644 index 000000000..631b54bad --- /dev/null +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -0,0 +1,102 @@ +(function() { + 'use strict' + function queryEditor() { + return { + restrict: 'E', + scope: { + 'query': '=', + 'lock': '=' + }, + template: '', + link: function($scope) { + $scope.editorOptions = { + mode: 'text/x-sql', + lineWrapping: true, + lineNumbers: true, + readOnly: false, + matchBrackets: true, + autoCloseBrackets: true + }; + + $scope.$watch('lock', function(locked) { + $scope.editorOptions.readOnly = locked ? 'nocursor' : false; + }); + } + } + } + + function queryFormatter($http) { + return { + restrict: 'E', + scope: { + 'query': '=', + 'lock': '=' + }, + template: '', + link: function($scope) { + $scope.formatQuery = function formatQuery() { + $scope.lock = true; + $http.post('/api/queries/format', { + 'query': $scope.query.query + }).success(function (response) { + $scope.query.query = response; + }).finally(function () { + $scope.lock = false; + }); + }; + } + } + } + + function queryRefreshSelect() { + return { + restrict: 'E', + template: '\ + ', + link: function($scope) { + $scope.refreshOptions = [ + { + value: -1, + name: 'No Refresh' + }, + { + value: 60, + name: 'Every minute' + }, + ] + + _.each(_.range(1, 13), function (i) { + $scope.refreshOptions.push({ + value: i * 3600, + name: 'Every ' + i + 'h' + }); + }) + + $scope.refreshOptions.push({ + value: 24 * 3600, + name: 'Every 24h' + }); + $scope.refreshOptions.push({ + value: 7 * 24 * 3600, + name: 'Once a week' + }); + } + + } + } + + angular.module('redash.directives') + .directive('queryRefreshSelect', queryRefreshSelect) + .directive('queryEditor', queryEditor) + .directive('queryFormatter', ['$http', queryFormatter]); +})(); \ No newline at end of file diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index d8d4df811..01c55615e 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -60,7 +60,7 @@ - +

      diff --git a/rd_ui/app/views/queryview.html b/rd_ui/app/views/queryview.html deleted file mode 100644 index 3bb5c54fb..000000000 --- a/rd_ui/app/views/queryview.html +++ /dev/null @@ -1,183 +0,0 @@ - -
      -
      -
      -
      -
      -

      - -

      -

      - - - -

      -
      -
      - -
      -
      - - - -
      -

      - Last update - - - -   - Created By - {{query.user.name}} - You -   - Runtime - {{queryResult.getRuntime() | durationHumanize}} - Running… -   - Rows - {{queryResult.getData().length}} -

      -

      - - Hide Source - View Source - -

      -
      -
      -
      - -
      - -
      -
      -
      -

      - - - - - - - -

      -
      - - -

      - -

      -
      -
      - -
      - -
      -
      -

      - - Last update - - - -

      -

      - - Created By - {{query.user.name}} - You -

      -

      - - Runtime - {{queryResult.getRuntime() | durationHumanize}} - Running… -

      -

      - - Rows {{queryResult.getData().length}} -

      -

      - - Refresh Interval - -

      - -

      - - Data Source - -

      - -
      - -

      - - - Download Dataset - -

      -
      - -
      - -
      - Executing query… - -
      -
      - Query in queue… - -
      -
      Error running query: {{queryResult.getError()}}
      - - -
      -
      -
      -
        - - - - × - - -
      -
      -
      -
      -
      -
      - - -
      - - - -
      - - -
      - -
      - - -
      -
      -
      -
      -
      -
      -
      From 5d03ce6b50c5046c9da56c59dcd3e7868e351fea Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 19 Mar 2014 14:18:02 +0200 Subject: [PATCH 426/540] [#138] QueryViewCtrl cleanup and formatting --- rd_ui/app/scripts/controllers/query_edit.js | 5 +- rd_ui/app/scripts/controllers/query_view.js | 195 ++++++++++---------- rd_ui/app/views/query.html | 33 ++-- redash/controllers.py | 2 +- 4 files changed, 116 insertions(+), 119 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index 00a59bca4..981d49efe 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -5,12 +5,15 @@ var pristineHash = ""; var leavingPageText = "You will lose your changes if you leave"; + // controller inheritance $controller('QueryViewCtrl', {$scope: $scope}); + $scope.sourceMode = true; + $scope.dirty = undefined; $scope.isNewQuery = false; - $scope.isSourceVisible = true; + $scope.canEdit = currentUser.canEdit($scope.query); $scope.newVisualization = undefined; diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 801873b31..784461e96 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,106 +1,103 @@ -(function () { - 'use strict'; +(function() { + 'use strict'; - function QueryViewCtrl($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) { - var DEFAULT_TAB = 'table'; - var route = $route.current; + function QueryViewCtrl($scope, $route, $location, notifications, Query) { + var DEFAULT_TAB = 'table'; - $scope.query = route.locals.query; - $scope.queryResult = $scope.query.getQueryResult(); - $scope.queryExecuting = false; + $scope.query = $route.current.locals.query; + $scope.queryResult = $scope.query.getQueryResult(); + $scope.queryExecuting = false; - $scope.isOwner = currentUser.canEdit($scope.query); - $scope.canEdit = $scope.isSourceVisible && $scope.isOwner; - $scope.canFork = true; + $scope.isQueryOwner = currentUser.id === $scope.query.user.id; - $scope.sourceHref = getQuerySourceUrl(); - - function getQuerySourceUrl(queryId) { - return '/queries/' + $scope.query.id + '/source#' + $location.hash(); - }; - - $scope.lockButton = function (lock) { - $scope.queryExecuting = lock; - }; - - - - $scope.duplicateQuery = function () { - var oldId = $scope.query.id; - $scope.query.id = null; - $scope.query.ttl = -1; - - $scope.saveQuery(true, oldId); - }; - - $scope.executeQuery = function () { - $scope.queryResult = $scope.query.getQueryResult(0); - $scope.lockButton(true); - $scope.cancelling = false; - }; - - $scope.cancelExecution = function () { - $scope.cancelling = true; - $scope.queryResult.cancelExecution(); - }; - - $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) { - if (newError == undefined) { - return; - } - - if (oldError == undefined && newError != undefined) { - $scope.lockButton(false); - } - }); - - $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) { - if (!data) { - return; - } - - $scope.filters = $scope.queryResult.getFilters(); - - if ($scope.queryResult.getId() == null) { - $scope.dataUri = ""; - } else { - $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv'; - $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv"; - } - }); - - $scope.$watch("queryResult && queryResult.getStatus()", function (status) { - if (!status) { - return; - } - - if (status == "done") { - if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() && - $scope.query.query_hash == $scope.queryResult.query_result.query_hash) { - Query.save({ - 'id': $scope.query.id, - 'latest_query_data_id': $scope.queryResult.getId() - }) - } - $scope.query.latest_query_data_id = $scope.queryResult.getId(); - - notifications.showNotification("re:dash", $scope.query.name + " updated."); - - $scope.lockButton(false); - } - }); - - $scope.$watch('query.name', function () { - $scope.$parent.pageTitle = $scope.query.name; - }); - $scope.$watch(function () { - return $location.hash() - }, function (hash) { - $scope.selectedTab = hash || DEFAULT_TAB; - $scope.sourceHref = getQuerySourceUrl(); - }); + $scope.lockButton = function(lock) { + $scope.queryExecuting = lock; }; - angular.module('redash.controllers') - .controller('QueryViewCtrl', ['$scope', '$window', '$route', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryViewCtrl]); + $scope.duplicateQuery = function() { + var oldId = $scope.query.id; + $scope.query.id = null; + $scope.query.ttl = -1; + + $scope.saveQuery(true, oldId); + }; + + $scope.executeQuery = function() { + $scope.queryResult = $scope.query.getQueryResult(0); + $scope.lockButton(true); + $scope.cancelling = false; + }; + + $scope.cancelExecution = function() { + $scope.cancelling = true; + $scope.queryResult.cancelExecution(); + }; + + $scope.$watch('queryResult && queryResult.getError()', + function(newError, oldError) { + if (newError == undefined) { + return; + } + + if (oldError == undefined && newError != undefined) { + $scope.lockButton(false); + } + }); + + $scope.$watch('queryResult && queryResult.getData()', + function(data, oldData) { + if (!data) { + return; + } + + if ($scope.queryResult.getId() == null) { + $scope.dataUri = ""; + } else { + $scope.dataUri = + '/api/queries/' + $scope.query.id + '/results/' + + $scope.queryResult.getId() + '.csv'; + + $scope.dataFilename = + $scope.query.name.replace(" ", "_") + + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + + ".csv"; + } + }); + + $scope.$watch("queryResult && queryResult.getStatus()", function(status) { + if (!status) { + return; + } + + if (status == "done") { + if ($scope.query.id && + $scope.query.latest_query_data_id != $scope.queryResult.getId() && + $scope.query.query_hash == $scope.queryResult.query_result.query_hash) { + Query.save({ + 'id': $scope.query.id, + 'latest_query_data_id': $scope.queryResult.getId() + }) + } + $scope.query.latest_query_data_id = $scope.queryResult.getId(); + + notifications.showNotification("re:dash", $scope.query.name + " updated."); + + $scope.lockButton(false); + } + }); + + $scope.$watch('query.name', function() { + $scope.$parent.pageTitle = $scope.query.name; + }); + + $scope.$watch(function() { + return $location.hash() + }, function(hash) { + $scope.selectedTab = hash || DEFAULT_TAB; + }); + }; + + angular.module('redash.controllers') + .controller('QueryViewCtrl', + ['$scope', '$route', '$location', 'notifications', 'Query', QueryViewCtrl]); })(); \ No newline at end of file diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 01c55615e..259f9e407 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -5,24 +5,21 @@

      - +

      - +

      +
      - + Show Source + Hide Source
      - -

      Last update @@ -31,8 +28,8 @@   Created By - {{query.user.name}} - You + {{query.user.name}} + You   Runtime {{queryResult.getRuntime() | durationHumanize}} @@ -43,8 +40,8 @@

      - Hide Source - View Source + Hide Source + View Source

      @@ -55,14 +52,14 @@
      -
      +

      - @@ -74,10 +71,10 @@

      -

      +

      -
      +
      @@ -94,8 +91,8 @@

      Created By - {{query.user.name}} - You + {{query.user.name}} + You

      diff --git a/redash/controllers.py b/redash/controllers.py index bd10c6d20..eefd57168 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -324,7 +324,7 @@ class QueryResultListAPI(BaseResource): @require_permission('execute_query') def post(self): params = request.json - + models.ActivityLog( user=self.current_user, type=models.ActivityLog.QUERY_EXECUTION, From 7616738fc600dc44f5f54d7547169bc13d4a846a Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 19 Mar 2014 17:31:32 +0200 Subject: [PATCH 427/540] [#138] QueryEditCtrl cleanup --- rd_ui/app/scripts/app.js | 22 +- rd_ui/app/scripts/controllers/query_edit.js | 203 ++++++++---------- rd_ui/app/scripts/controllers/query_view.js | 1 + rd_ui/app/scripts/directives/directives.js | 18 ++ .../scripts/directives/query_directives.js | 20 +- rd_ui/app/views/query.html | 26 +-- 6 files changed, 160 insertions(+), 130 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 1b7006523..7ff54b51a 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -17,6 +17,15 @@ angular.module('redash', [ ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', function($routeProvider, $locationProvider, $compileProvider, growlProvider) { + function newQuery(Query) { + return new Query({ + query: "", + name: "New Query", + ttl: -1, + user: currentUser + }); + } + function getQuery(Query, $q, $route) { var defer = $q.defer(); @@ -42,6 +51,14 @@ angular.module('redash', [ controller: 'QueriesCtrl', reloadOnSearch: false }); + $routeProvider.when('/queries/new', { + templateUrl: '/views/query.html', + controller: 'QueryEditCtrl', + reloadOnSearch: false, + resolve: { + 'query': ['Query', newQuery] + } + }); $routeProvider.when('/queries/:queryId', { templateUrl: '/views/query.html', controller: 'QueryViewCtrl', @@ -50,11 +67,6 @@ angular.module('redash', [ 'query': ['Query', '$q', '$route', getQuery] } }); - $routeProvider.when('/queries/new', { - templateUrl: '/views/query.html', - controller: 'QueryEditCtrl', - reloadOnSearch: false - }); $routeProvider.when('/queries/:queryId/source', { templateUrl: '/views/query.html', controller: 'QueryEditCtrl', diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index 981d49efe..7e257b90b 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -1,125 +1,106 @@ -(function () { - 'use strict'; +(function() { + 'use strict'; - function QueryEditCtrl($controller, $scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) { - var pristineHash = ""; - var leavingPageText = "You will lose your changes if you leave"; + function QueryEditCtrl($controller, $scope, $window, $route, $location, growl, Query, Visualization) { + var pristineHash = ""; + var leavingPageText = "You will lose your changes if you leave"; + var isNewQuery = !$route.current.locals.query.id; - // controller inheritance - $controller('QueryViewCtrl', {$scope: $scope}); + // controller inheritance + $controller('QueryViewCtrl', {$scope: $scope}); - $scope.sourceMode = true; + $scope.sourceMode = true; + $scope.isDirty = undefined; - $scope.dirty = undefined; - $scope.isNewQuery = false; + $scope.canEdit = currentUser.canEdit($scope.query); - $scope.canEdit = currentUser.canEdit($scope.query); + $scope.newVisualization = undefined; - $scope.newVisualization = undefined; + $window.onbeforeunload = function() { + if ($scope.canEdit && $scope.isDirty) { + return leavingPageText; + } + } - $window.onbeforeunload = function () { - if ($scope.canEdit && $scope.dirty) { - return leavingPageText; - } + $scope.$on('$locationChangeStart', function(event, next, current) { + if (next.split("#")[0] == current.split("#")[0]) { + return; + } + + if (!$scope.canEdit) { + return; + } + + if ($scope.isDirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { + event.preventDefault(); + } else { + Mousetrap.unbind("meta+s"); + } + }); + + $scope.saveQuery = function(duplicate, oldId) { + if (!oldId) { + oldId = $scope.query.id; + } + + delete $scope.query.latest_query_data; + $scope.query.$save(function(q) { + pristineHash = q.getHash(); + $scope.isDirty = false; + + if (duplicate) { + growl.addSuccessMessage("Query forked"); + } else { + growl.addSuccessMessage("Query saved"); } - function getQuerySourceUrl(queryId) { - return '/queries/' + queryId + '/source#' + $location.hash(); - }; - - Mousetrap.bindGlobal("meta+s", function (e) { - e.preventDefault(); - - if ($scope.canEdit) { - $scope.saveQuery(); - } - }); - - $scope.$on('$locationChangeStart', function (event, next, current) { - if (next.split("#")[0] == current.split("#")[0]) { - return; - } - - if (!$scope.canEdit) { - return; - } - - if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { - event.preventDefault(); - } else { - Mousetrap.unbind("meta+s"); - } - }); - - $scope.saveQuery = function (duplicate, oldId) { - if (!oldId) { - oldId = $scope.query.id; - } - - delete $scope.query.latest_query_data; - $scope.query.$save(function (q) { - pristineHash = q.getHash(); - $scope.dirty = false; - - if (duplicate) { - growl.addSuccessMessage("Query forked"); - } else { - growl.addSuccessMessage("Query saved"); - } - - if (oldId != q.id) { - $location.url(getQuerySourceUrl(q.id)).replace(); - } - }, function (httpResponse) { - growl.addErrorMessage("Query could not be saved"); - }); - }; - - - // new query - if (!$route.current.locals.query) { - $scope.query = new Query({ - query: "", - name: "New Query", - ttl: -1, - user: currentUser - }); - $scope.lockButton(false); - $scope.isOwner = $scope.canEdit = true; - $scope.isNewQuery = true; - - var unbind = $scope.$watch('selectedTab == "add"', function (newPanel) { - if (newPanel && route.params.queryId == undefined) { - unbind(); - $scope.saveQuery(); - } - }); + if (oldId != q.id) { + $location.url($location.url().replace(oldId, q.id)).replace(); } - - $scope.$watch(function () { - return $scope.query.getHash(); - }, function (newHash) { - $scope.dirty = (newHash !== pristineHash); - }); - - - $scope.deleteVisualization = function ($e, vis) { - $e.preventDefault(); - if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { - Visualization.delete(vis); - if ($scope.selectedTab == vis.id) { - $scope.selectedTab = DEFAULT_TAB; - $location.hash($scope.selectedTab); - } - $scope.query.visualizations = - $scope.query.visualizations.filter(function (v) { - return vis.id !== v.id; - }); - } - }; + }, function(httpResponse) { + growl.addErrorMessage("Query could not be saved"); + }); }; - angular.module('redash.controllers') - .controller('QueryEditCtrl', ['$controller', '$scope', '$window', '$route', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryEditCtrl]); + $scope.deleteVisualization = function($e, vis) { + $e.preventDefault(); + if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { + Visualization.delete(vis); + if ($scope.selectedTab == vis.id) { + $scope.selectedTab = DEFAULT_TAB; + $location.hash($scope.selectedTab); + } + $scope.query.visualizations = + $scope.query.visualizations.filter(function(v) { + return vis.id !== v.id; + }); + } + }; + + + $scope.$watch(function() { + return $scope.query.getHash(); + }, function(newHash) { + $scope.isDirty = (newHash !== pristineHash); + }); + + if (isNewQuery) { + // $scope.lockButton(false); + + // save new query when creating a visualization + var unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { + if (newPanel && $route.current.params.queryId == undefined) { + unbind(); + $scope.saveQuery(); + } + }); + } + + }; + + angular.module('redash.controllers').controller('QueryEditCtrl', [ + '$controller', '$scope', '$window', '$route', '$location', 'growl', 'Query', + 'Visualization', QueryEditCtrl + ]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 784461e96..5c481192a 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -9,6 +9,7 @@ $scope.queryExecuting = false; $scope.isQueryOwner = currentUser.id === $scope.query.user.id; + $scope.canViewSource = currentUser.hasPermission('view_source'); $scope.lockButton = function(lock) { $scope.queryExecuting = lock; diff --git a/rd_ui/app/scripts/directives/directives.js b/rd_ui/app/scripts/directives/directives.js index da2b1f1de..5791adc3e 100644 --- a/rd_ui/app/scripts/directives/directives.js +++ b/rd_ui/app/scripts/directives/directives.js @@ -3,6 +3,24 @@ var directives = angular.module('redash.directives', []); + directives.directive('keyboardShortcut', function() { + return { + restrict: 'E', + replace: true, + template: 'foo', + scope: { + key: '@', + action: '=' + }, + link: function($scope) { + Mousetrap.bindGlobal($scope.key, function(e) { + e.preventDefault(); + $scope.action(); + }); + } + } + }); + directives.directive('rdTab', function() { return { restrict: 'E', diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 631b54bad..abd3569fe 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -1,5 +1,20 @@ (function() { 'use strict' + + function querySourceLink() { + return { + restrict: 'E', + template: '\ + Show Source\ + \ + Hide Source\ + \ + ' + } + } + function queryEditor() { return { restrict: 'E', @@ -58,7 +73,7 @@ return { restrict: 'E', template: '\ @@ -96,7 +111,8 @@ } angular.module('redash.directives') - .directive('queryRefreshSelect', queryRefreshSelect) + .directive('querySourceLink', querySourceLink) .directive('queryEditor', queryEditor) + .directive('queryRefreshSelect', queryRefreshSelect) .directive('queryFormatter', ['$http', queryFormatter]); })(); \ No newline at end of file diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 259f9e407..1b3cd07dd 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -1,5 +1,7 @@

      + +
      @@ -14,9 +16,10 @@

      -
      - Show Source - Hide Source +
      +
      + +
      @@ -39,10 +42,7 @@ {{queryResult.getData().length}}

      - - Hide Source - View Source - +

      @@ -64,17 +64,19 @@

      -

      - -

      -
      +
      +

      + +

      +
      +
      From f3411a46a583f02d286f24c4c17133a772b656c3 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 15:20:14 +0200 Subject: [PATCH 428/540] [#138] alert-unsaved-changes directive --- rd_ui/app/scripts/controllers/query_edit.js | 48 ++++----------------- rd_ui/app/scripts/directives/directives.js | 42 +++++++++++++++++- rd_ui/app/scripts/services/services.js | 4 -- rd_ui/app/views/query.html | 2 + 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index 7e257b90b..701fafb83 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -1,51 +1,25 @@ (function() { 'use strict'; - function QueryEditCtrl($controller, $scope, $window, $route, $location, growl, Query, Visualization) { - var pristineHash = ""; - var leavingPageText = "You will lose your changes if you leave"; + function QueryEditCtrl($controller, $scope, $route, $location, growl, Query, Visualization) { var isNewQuery = !$route.current.locals.query.id; // controller inheritance $controller('QueryViewCtrl', {$scope: $scope}); $scope.sourceMode = true; - $scope.isDirty = undefined; - + $scope.isDirty = false; $scope.canEdit = currentUser.canEdit($scope.query); $scope.newVisualization = undefined; - $window.onbeforeunload = function() { - if ($scope.canEdit && $scope.isDirty) { - return leavingPageText; - } - } - - $scope.$on('$locationChangeStart', function(event, next, current) { - if (next.split("#")[0] == current.split("#")[0]) { - return; - } - - if (!$scope.canEdit) { - return; - } - - if ($scope.isDirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) { - event.preventDefault(); - } else { - Mousetrap.unbind("meta+s"); - } - }); - $scope.saveQuery = function(duplicate, oldId) { if (!oldId) { oldId = $scope.query.id; } delete $scope.query.latest_query_data; - $scope.query.$save(function(q) { - pristineHash = q.getHash(); + $scope.query.$save(function(savedQuery) { $scope.isDirty = false; if (duplicate) { @@ -54,15 +28,14 @@ growl.addSuccessMessage("Query saved"); } - if (oldId != q.id) { - $location.url($location.url().replace(oldId, q.id)).replace(); + if (oldId != savedQuery.id) { + $location.url($location.url().replace(oldId, savedQuery.id)).replace(); } }, function(httpResponse) { growl.addErrorMessage("Query could not be saved"); }); }; - $scope.deleteVisualization = function($e, vis) { $e.preventDefault(); if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { @@ -78,16 +51,11 @@ } }; - - $scope.$watch(function() { - return $scope.query.getHash(); - }, function(newHash) { - $scope.isDirty = (newHash !== pristineHash); + $scope.$watch('query.query', function(current, prev) { + $scope.isDirty = (current !== prev); }); if (isNewQuery) { - // $scope.lockButton(false); - // save new query when creating a visualization var unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { if (newPanel && $route.current.params.queryId == undefined) { @@ -100,7 +68,7 @@ }; angular.module('redash.controllers').controller('QueryEditCtrl', [ - '$controller', '$scope', '$window', '$route', '$location', 'growl', 'Query', + '$controller', '$scope', '$route', '$location', 'growl', 'Query', 'Visualization', QueryEditCtrl ]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/directives/directives.js b/rd_ui/app/scripts/directives/directives.js index 5791adc3e..9de539e9f 100644 --- a/rd_ui/app/scripts/directives/directives.js +++ b/rd_ui/app/scripts/directives/directives.js @@ -3,11 +3,47 @@ var directives = angular.module('redash.directives', []); + directives.directive('alertUnsavedChanges', ['$window', function($window) { + return { + restrict: 'E', + replace: true, + scope: { + 'isDirty': '=' + }, + link: function($scope) { + var + + unloadMessage = "You will lose your changes if you leave", + confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?", + + // store original handler (if any) + _onbeforeunload = $window.onbeforeunload; + + $window.onbeforeunload = function() { + return $scope.isDirty ? unloadMessage : null; + } + + $scope.$on('$locationChangeStart', function(event, next, current) { + if (next.split("#")[0] == current.split("#")[0]) { + return; + } + + if ($scope.isDirty && !confirm(confirmMessage)) { + event.preventDefault(); + } + }); + + $scope.$on('$destroy', function() { + $window.onbeforeunload = _onbeforeunload; + }); + } + } + }]); + directives.directive('keyboardShortcut', function() { return { restrict: 'E', replace: true, - template: 'foo', scope: { key: '@', action: '=' @@ -17,6 +53,10 @@ e.preventDefault(); $scope.action(); }); + + $scope.$on('$destroy', function() { + Mousetrap.unbind($scope.key); + }); } } }); diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 225295c2c..8d5a5e5fa 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -321,10 +321,6 @@ return queryResult; }; - Query.prototype.getHash = function() { - return this.query; - }; - return Query; }; diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 1b3cd07dd..4fc58de93 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -1,5 +1,7 @@
      + +
      From 6a1c5aeae7c9d586e135644da7da7b810ac1fa14 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 15:37:52 +0200 Subject: [PATCH 429/540] [#138] move saveQuery to ViewCtrl --- rd_ui/app/scripts/controllers/query_edit.js | 23 ------------------ rd_ui/app/scripts/controllers/query_view.js | 27 +++++++++++++++++++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index 701fafb83..d49eddb79 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -13,29 +13,6 @@ $scope.newVisualization = undefined; - $scope.saveQuery = function(duplicate, oldId) { - if (!oldId) { - oldId = $scope.query.id; - } - - delete $scope.query.latest_query_data; - $scope.query.$save(function(savedQuery) { - $scope.isDirty = false; - - if (duplicate) { - growl.addSuccessMessage("Query forked"); - } else { - growl.addSuccessMessage("Query saved"); - } - - if (oldId != savedQuery.id) { - $location.url($location.url().replace(oldId, savedQuery.id)).replace(); - } - }, function(httpResponse) { - growl.addErrorMessage("Query could not be saved"); - }); - }; - $scope.deleteVisualization = function($e, vis) { $e.preventDefault(); if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 5c481192a..16e015cd6 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - function QueryViewCtrl($scope, $route, $location, notifications, Query) { + function QueryViewCtrl($scope, $route, $location, notifications, Query, growl) { var DEFAULT_TAB = 'table'; $scope.query = $route.current.locals.query; @@ -15,6 +15,29 @@ $scope.queryExecuting = lock; }; + $scope.saveQuery = function(duplicate, oldId) { + if (!oldId) { + oldId = $scope.query.id; + } + + delete $scope.query.latest_query_data; + $scope.query.$save(function(savedQuery) { + $scope.isDirty = false; + + if (duplicate) { + growl.addSuccessMessage("Query forked"); + } else { + growl.addSuccessMessage("Query saved"); + } + + if (oldId != savedQuery.id) { + $location.url($location.url().replace(oldId, savedQuery.id)).replace(); + } + }, function(httpResponse) { + growl.addErrorMessage("Query could not be saved"); + }); + }; + $scope.duplicateQuery = function() { var oldId = $scope.query.id; $scope.query.id = null; @@ -100,5 +123,5 @@ angular.module('redash.controllers') .controller('QueryViewCtrl', - ['$scope', '$route', '$location', 'notifications', 'Query', QueryViewCtrl]); + ['$scope', '$route', '$location', 'notifications', 'Query', 'growl', QueryViewCtrl]); })(); \ No newline at end of file From cddf69e422ffb923bb217bf2d34303d325f449f9 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 15:42:52 +0200 Subject: [PATCH 430/540] [#138] rebase fixes (cherry pick f3d4635) --- rd_ui/app/scripts/controllers/query_view.js | 2 ++ rd_ui/app/views/query.html | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 16e015cd6..1b684a86c 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -74,6 +74,8 @@ return; } + $scope.filters = $scope.queryResult.getFilters(); + if ($scope.queryResult.getId() == null) { $scope.dataUri = ""; } else { diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 4fc58de93..5b5b37d9b 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -152,7 +152,11 @@
      - +
      + + +
      +
      From 34fb58d403d4c9b1e863a9556d1c3ee5746fc9bb Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 15:59:12 +0200 Subject: [PATCH 431/540] typo: cancelInterval -> clearInterval --- rd_ui/app/scripts/directives/directives.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/directives/directives.js b/rd_ui/app/scripts/directives/directives.js index 9de539e9f..c531b4b12 100644 --- a/rd_ui/app/scripts/directives/directives.js +++ b/rd_ui/app/scripts/directives/directives.js @@ -377,7 +377,7 @@ $scope.$on('$destroy', function () { if (currentTimer) { - cancelInterval(currentTimer); + clearInterval(currentTimer); currentTimer = null; } }); From c4cbe06c12870e4f65f926110860d24c75ea04dd Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 16:56:36 +0200 Subject: [PATCH 432/540] [#138] Query.newQuery --- rd_ui/app/scripts/app.js | 13 +++---------- rd_ui/app/scripts/services/services.js | 9 +++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 7ff54b51a..9ed52fba5 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -17,15 +17,6 @@ angular.module('redash', [ ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', function($routeProvider, $locationProvider, $compileProvider, growlProvider) { - function newQuery(Query) { - return new Query({ - query: "", - name: "New Query", - ttl: -1, - user: currentUser - }); - } - function getQuery(Query, $q, $route) { var defer = $q.defer(); @@ -56,7 +47,9 @@ angular.module('redash', [ controller: 'QueryEditCtrl', reloadOnSearch: false, resolve: { - 'query': ['Query', newQuery] + 'query': ['Query', function newQuery(Query) { + return Query.newQuery(); + }] } }); $routeProvider.when('/queries/:queryId', { diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 8d5a5e5fa..19fdcb17d 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -304,6 +304,15 @@ var Query = function ($resource, QueryResult, DataSource) { var Query = $resource('/api/queries/:id', {id: '@id'}); + Query.newQuery = function() { + return new Query({ + query: "", + name: "New Query", + ttl: -1, + user: currentUser + }); + }; + Query.prototype.getQueryResult = function(ttl) { if (ttl == undefined) { ttl = this.ttl; From ca415c50ad247bd7d3c672dd87ef5cbaeac7f375 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Sun, 23 Mar 2014 17:06:23 +0200 Subject: [PATCH 433/540] [#138] store original query text to detect changes --- rd_ui/app/scripts/controllers/query_edit.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index d49eddb79..a904e6e38 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -2,11 +2,13 @@ 'use strict'; function QueryEditCtrl($controller, $scope, $route, $location, growl, Query, Visualization) { - var isNewQuery = !$route.current.locals.query.id; - - // controller inheritance + // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); + var + _queryText = $scope.query.query, + isNewQuery = !$route.current.locals.query.id; + $scope.sourceMode = true; $scope.isDirty = false; $scope.canEdit = currentUser.canEdit($scope.query); @@ -28,8 +30,8 @@ } }; - $scope.$watch('query.query', function(current, prev) { - $scope.isDirty = (current !== prev); + $scope.$watch('query.query', function(newQueryText) { + $scope.isDirty = (newQueryText !== _queryText); }); if (isNewQuery) { From e0af1f20af1c2082913ac58f6fb5f8262dc4f7ef Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 24 Mar 2014 11:31:56 +0200 Subject: [PATCH 434/540] [#138] cherry pick rebase conflicts: 366cdbf616 Remove reference to query result when changing data source 872cee2228 Unless data source set already, set it to the first one. 8ae41c0b6a Show query's data source. --- rd_ui/app/scripts/controllers/query_view.js | 20 ++++++++++++++++++-- rd_ui/app/views/query.html | 6 ++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 1b684a86c..3ac3b4fa3 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - function QueryViewCtrl($scope, $route, $location, notifications, Query, growl) { + function QueryViewCtrl($scope, $route, $location, notifications, growl, Query, DataSource) { var DEFAULT_TAB = 'table'; $scope.query = $route.current.locals.query; @@ -11,6 +11,10 @@ $scope.isQueryOwner = currentUser.id === $scope.query.user.id; $scope.canViewSource = currentUser.hasPermission('view_source'); + $scope.dataSources = DataSource.get(function(dataSources) { + $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id; + }); + $scope.lockButton = function(lock) { $scope.queryExecuting = lock; }; @@ -57,6 +61,18 @@ $scope.queryResult.cancelExecution(); }; + $scope.updateDataSource = function() { + $scope.query.latest_query_data = null; + $scope.query.latest_query_data_id = null; + Query.save({ + 'id': $scope.query.id, + 'data_source_id': $scope.query.data_source_id, + 'latest_query_data_id': null + }); + + $scope.executeQuery(); + }; + $scope.$watch('queryResult && queryResult.getError()', function(newError, oldError) { if (newError == undefined) { @@ -125,5 +141,5 @@ angular.module('redash.controllers') .controller('QueryViewCtrl', - ['$scope', '$route', '$location', 'notifications', 'Query', 'growl', QueryViewCtrl]); + ['$scope', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]); })(); \ No newline at end of file diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 5b5b37d9b..462f0bef0 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -114,6 +114,12 @@

      +

      + + Data Source + +

      +

      From 16a83f6134ff2ac1aedeb7c919528785c6a06005 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 24 Mar 2014 12:07:00 +0200 Subject: [PATCH 435/540] getQueryResult only if query.data_source_id exists --- rd_ui/app/scripts/services/services.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 19fdcb17d..98050408e 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -323,7 +323,7 @@ queryResult = new QueryResult({'query_result': this.latest_query_data}); } else if (this.latest_query_data_id && ttl != 0) { queryResult = QueryResult.getById(this.latest_query_data_id); - } else { + } else if (this.data_source_id) { queryResult = QueryResult.get(this.data_source_id, this.query, ttl); } From 977193b0094ba4bf756b76517c6553264ffd5e68 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 24 Mar 2014 13:43:37 +0200 Subject: [PATCH 436/540] fix getColumns failures when QueryResult has no data --- rd_ui/app/scripts/services/services.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 98050408e..74ffb000f 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -182,10 +182,10 @@ }; QueryResult.prototype.getColumns = function () { - if (this.columns == undefined) { + if (this.columns == undefined && this.query_result.data) { this.columns = _.map(this.query_result.data.columns, function(v) { return v.name; - }) + }); } return this.columns; From 21ad5bbb4a363a898d41b7093e662f8b6376d3b0 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 24 Mar 2014 15:28:05 +0200 Subject: [PATCH 437/540] [#138] simplify saveQuery, drop $route dependency --- rd_ui/app/scripts/controllers/query_edit.js | 10 +++---- rd_ui/app/scripts/controllers/query_view.js | 30 ++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index a904e6e38..eb6c31f0a 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -1,13 +1,13 @@ (function() { 'use strict'; - function QueryEditCtrl($controller, $scope, $route, $location, growl, Query, Visualization) { + function QueryEditCtrl($controller, $scope, $location, growl, Query, Visualization) { // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); var _queryText = $scope.query.query, - isNewQuery = !$route.current.locals.query.id; + isNewQuery = !$scope.query.id; $scope.sourceMode = true; $scope.isDirty = false; @@ -36,8 +36,8 @@ if (isNewQuery) { // save new query when creating a visualization - var unbind = $scope.$watch('selectedTab == "add"', function(newPanel) { - if (newPanel && $route.current.params.queryId == undefined) { + var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) { + if (triggerSave) { unbind(); $scope.saveQuery(); } @@ -47,7 +47,7 @@ }; angular.module('redash.controllers').controller('QueryEditCtrl', [ - '$controller', '$scope', '$route', '$location', 'growl', 'Query', + '$controller', '$scope', '$location', 'growl', 'Query', 'Visualization', QueryEditCtrl ]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 3ac3b4fa3..42ab58b3b 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -19,35 +19,35 @@ $scope.queryExecuting = lock; }; - $scope.saveQuery = function(duplicate, oldId) { - if (!oldId) { - oldId = $scope.query.id; + $scope.saveQuery = function(duplicate) { + var + successMessage = "Query saved", + oldId = $scope.query.id; + + if (duplicate) { + successMessage = "Query forked"; + $scope.query.id = null; + $scope.query.ttl = -1; } delete $scope.query.latest_query_data; + $scope.query.$save(function(savedQuery) { $scope.isDirty = false; - if (duplicate) { - growl.addSuccessMessage("Query forked"); - } else { - growl.addSuccessMessage("Query saved"); + if (oldId != savedQuery.id) { + // redirect to new/duplicated query page + $location.url('/queries/' + savedQuery.id + '/source#' + $location.hash()).replace(); } - if (oldId != savedQuery.id) { - $location.url($location.url().replace(oldId, savedQuery.id)).replace(); - } + growl.addSuccessMessage(successMessage); }, function(httpResponse) { growl.addErrorMessage("Query could not be saved"); }); }; $scope.duplicateQuery = function() { - var oldId = $scope.query.id; - $scope.query.id = null; - $scope.query.ttl = -1; - - $scope.saveQuery(true, oldId); + $scope.saveQuery(true); }; $scope.executeQuery = function() { From bbe17f3a0971d2ebea37926d46bc956d76e1437c Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Mon, 24 Mar 2014 16:13:59 +0200 Subject: [PATCH 438/540] [#138] fix ui-codemirror bug when used as directive --- rd_ui/app/scripts/directives/query_directives.js | 11 +++++------ rd_ui/app/views/query.html | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index abd3569fe..288c64eb4 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -45,10 +45,9 @@ function queryFormatter($http) { return { restrict: 'E', - scope: { - 'query': '=', - 'lock': '=' - }, + // don't create new scope to avoid ui-codemirror bug + // seehttps://github.com/angular-ui/ui-codemirror/pull/37 + scope: false, template: '', link: function($scope) { $scope.formatQuery = function formatQuery() { - $scope.lock = true; + $scope.queryExecuting = true; $http.post('/api/queries/format', { 'query': $scope.query.query }).success(function (response) { $scope.query.query = response; }).finally(function () { - $scope.lock = false; + $scope.queryExecuting = false; }); }; } diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 462f0bef0..39d8fb00b 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -59,7 +59,7 @@ - +

      From 5ad2bd048cbf64920b7f17162d9e9a209c3c8545 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 25 Mar 2014 17:12:49 +0200 Subject: [PATCH 451/540] [#90] perf: don't render (ngIf) the table visualization tab instead of hiding (ngHide) --- rd_ui/app/views/query.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 4c1e282eb..64f4d4d66 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -148,7 +148,7 @@
        - + × From 23760ffa86c9b24498d8c0dae1b46f51e0898e87 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 25 Mar 2014 17:29:19 +0200 Subject: [PATCH 452/540] [#90] switch to new visualization tab on save --- rd_ui/app/scripts/controllers/query_view.js | 5 +++++ rd_ui/app/scripts/visualizations/base.js | 5 ++++- rd_ui/app/views/query.html | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 1157a670b..8a757ce23 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -89,6 +89,11 @@ $scope.executeQuery(); }; + $scope.setVisualizationTab = function (visualization) { + $scope.selectedTab = visualization.id; + $location.hash(visualization.id); + }; + $scope.$watch('query.name', function() { $scope.$parent.pageTitle = $scope.query.name; }); diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index c60a1573a..acc90ef08 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -99,7 +99,8 @@ scope: { query: '=', queryResult: '=', - visualization: '=?' + visualization: '=?', + onNewSuccess: '=?' }, link: function (scope, element, attrs) { scope.visTypes = Visualization.visualizationTypes; @@ -145,7 +146,9 @@ if (index > -1) { scope.query.visualizations[index] = result; } else { + // new visualization scope.query.visualizations.push(result); + scope.onNewSuccess && scope.onNewSuccess(result); } }, function error() { growl.addErrorMessage("Visualization could not be saved"); diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 64f4d4d66..113a38f8f 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -171,7 +171,7 @@
        - +
      From 11ba93cc8022c3e1de068368f6c207a06bd4aa98 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Tue, 25 Mar 2014 17:58:38 +0200 Subject: [PATCH 453/540] [#90] query links in widget title --- rd_ui/app/scripts/controllers/controllers.js | 5 ----- .../scripts/directives/query_directives.js | 21 +++++++++++++++++++ rd_ui/app/views/dashboard.html | 4 ++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index fb776e639..fae5d8f35 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -58,11 +58,6 @@ }); }; - $scope.open = function(query, visualization) { - $location.path('/queries/' + query.id); - $location.hash(visualization.id); - } - $scope.query = new Query($scope.widget.visualization.query); $scope.queryResult = $scope.query.getQueryResult(); diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 288c64eb4..ba06e7e36 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -1,6 +1,26 @@ (function() { 'use strict' + function queryLink() { + return { + restrict: 'E', + scope: { + 'query': '=?', + 'visualization': '=?' + }, + template: '{{query.name}}', + link: function(scope, element) { + scope.query = scope.query || scope.$parent.query; + + var link = '/queries/' + scope.query.id; + if (scope.visualization) { + link += '#' + scope.visualization.id; + } + element.find('a').attr('href', link); + } + } + } + function querySourceLink() { return { restrict: 'E', @@ -110,6 +130,7 @@ } angular.module('redash.directives') + .directive('queryLink', queryLink) .directive('querySourceLink', querySourceLink) .directive('queryEditor', queryEditor) .directive('queryRefreshSelect', queryRefreshSelect) diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 9ee76f7f7..8d13b7447 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -23,9 +23,9 @@
      -

      +

      - +

      From 84b0590ec566fea4f7058727165bab5bb4dae5c4 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 12:12:30 +0200 Subject: [PATCH 454/540] move DashboardCtrl and WidgetCtrl to dashboard.js --- rd_ui/app/index.html | 1 + rd_ui/app/scripts/controllers/controllers.js | 70 ------------------ rd_ui/app/scripts/controllers/dashboard.js | 78 ++++++++++++++++++++ 3 files changed, 79 insertions(+), 70 deletions(-) create mode 100644 rd_ui/app/scripts/controllers/dashboard.js diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index b8ec71a28..668fe20bd 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -120,6 +120,7 @@ + diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index fae5d8f35..d68dfe92c 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -1,72 +1,4 @@ (function () { - var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) { - $scope.refreshEnabled = false; - $scope.refreshRate = 60; - $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { - $scope.$parent.pageTitle = dashboard.name; - }); - - var autoRefresh = function() { - if ($scope.refreshEnabled) { - $timeout(function() { - Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) { - var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id'); - - _.each($scope.dashboard.widgets, function(row) { - _.each(row, function(widget, i) { - var newWidget = newWidgets[widget.id]; - if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) { - row[i] = newWidget[0]; - } - }); - }); - - autoRefresh(); - }); - - }, $scope.refreshRate); - }; - } - - $scope.triggerRefresh = function(){ - $scope.refreshEnabled = !$scope.refreshEnabled; - - if ($scope.refreshEnabled) { - var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) { - return widget.visualization.query.ttl; - }).visualization.query.ttl; - - $scope.refreshRate = _.max([120, refreshRate * 2])*1000; - - autoRefresh(); - } - }; - }; - - var WidgetCtrl = function ($scope, $http, $location, Query) { - $scope.deleteWidget = function() { - if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) { - return; - } - - $http.delete('/api/widgets/' + $scope.widget.id).success(function() { - $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) { - return _.filter(row, function(widget) { - return widget.id != $scope.widget.id; - }) - }); - }); - }; - - $scope.query = new Query($scope.widget.visualization.query); - $scope.queryResult = $scope.query.getQueryResult(); - - $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString(); - $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow(); - - $scope.updateTime = ''; - } - var QueriesCtrl = function($scope, $http, $location, $filter, Query) { $scope.$parent.pageTitle = "All Queries"; $scope.gridConfig = { @@ -217,8 +149,6 @@ } angular.module('redash.controllers', []) - .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) - .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js new file mode 100644 index 000000000..e427ed954 --- /dev/null +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -0,0 +1,78 @@ +(function() { + var DashboardCtrl = function($scope, $routeParams, $http, $timeout, Dashboard) { + $scope.refreshEnabled = false; + $scope.refreshRate = 60; + $scope.dashboard = Dashboard.get({ + slug: $routeParams.dashboardSlug + }, function(dashboard) { + $scope.$parent.pageTitle = dashboard.name; + }); + + var autoRefresh = function() { + if ($scope.refreshEnabled) { + $timeout(function() { + Dashboard.get({ + slug: $routeParams.dashboardSlug + }, function(dashboard) { + var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id'); + + _.each($scope.dashboard.widgets, function(row) { + _.each(row, function(widget, i) { + var newWidget = newWidgets[widget.id]; + if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) { + row[i] = newWidget[0]; + } + }); + }); + + autoRefresh(); + }); + + }, $scope.refreshRate); + }; + } + + $scope.triggerRefresh = function() { + $scope.refreshEnabled = !$scope.refreshEnabled; + + if ($scope.refreshEnabled) { + var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) { + return widget.visualization.query.ttl; + }).visualization.query.ttl; + + $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000; + + autoRefresh(); + } + }; + }; + + var WidgetCtrl = function($scope, $http, $location, Query) { + $scope.deleteWidget = function() { + if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) { + return; + } + + $http.delete('/api/widgets/' + $scope.widget.id).success(function() { + $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) { + return _.filter(row, function(widget) { + return widget.id != $scope.widget.id; + }) + }); + }); + }; + + $scope.query = new Query($scope.widget.visualization.query); + $scope.queryResult = $scope.query.getQueryResult(); + + $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString(); + $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow(); + + $scope.updateTime = ''; + }; + + angular.module('redash.controllers') + .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) + .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) + +})(); \ No newline at end of file From 7ac76c2996787c97dd1d7a3280d647569c6d9bf8 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 12:18:50 +0200 Subject: [PATCH 455/540] dashboard_directives.js --- rd_ui/app/index.html | 1 + .../directives/dashboard_directives.js | 183 ++++++++++++++++++ rd_ui/app/scripts/directives/directives.js | 163 ---------------- 3 files changed, 184 insertions(+), 163 deletions(-) create mode 100644 rd_ui/app/scripts/directives/dashboard_directives.js diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 668fe20bd..c300dfcd1 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -131,6 +131,7 @@ + diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js new file mode 100644 index 000000000..ff707348a --- /dev/null +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -0,0 +1,183 @@ +(function() { + 'use strict' + + var directives = angular.module('redash.directives'); + + directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', + function($http, $location, $timeout, Dashboard) { + return { + restrict: 'E', + scope: { + dashboard: '=' + }, + templateUrl: '/views/edit_dashboard.html', + replace: true, + link: function($scope, element, attrs) { + var gridster = element.find(".gridster ul").gridster({ + widget_margins: [5, 5], + widget_base_dimensions: [260, 100], + min_cols: 2, + max_cols: 2, + serialize_params: function($w, wgd) { + return { + col: wgd.col, + row: wgd.row, + id: $w.data('widget-id') + } + } + }).data('gridster'); + + var gsItemTemplate = '
    • ' + + '
      {name}' + + '
      • '; + + $scope.$watch('dashboard.widgets', function(widgets) { + $timeout(function() { + gridster.remove_all_widgets(); + + if (widgets && widgets.length) { + var layout = []; + + _.each(widgets, function(row, rowIndex) { + _.each(row, function(widget, colIndex) { + layout.push({ + id: widget.id, + col: colIndex + 1, + row: rowIndex + 1, + ySize: 1, + xSize: widget.width, + name: widget.visualization.query.name + }); + }); + }); + + _.each(layout, function(item) { + var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); + gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); + + }); + } + }); + }, true); + + $scope.saveDashboard = function() { + $scope.saveInProgress = true; + // TODO: we should use the dashboard service here. + if ($scope.dashboard.id) { + var positions = $(element).find('.gridster ul').data('gridster').serialize(); + var layout = []; + _.each(_.sortBy(positions, function(pos) { + return pos.row * 10 + pos.col; + }), function(pos) { + var row = pos.row - 1; + var col = pos.col - 1; + layout[row] = layout[row] || []; + if (col > 0 && layout[row][col - 1] == undefined) { + layout[row][col - 1] = pos.id; + } else { + layout[row][col] = pos.id; + } + + }); + $scope.dashboard.layout = layout; + + layout = JSON.stringify(layout); + $http.post('/api/dashboards/' + $scope.dashboard.id, { + 'name': $scope.dashboard.name, + 'layout': layout + }).success(function(response) { + $scope.dashboard = new Dashboard(response); + $scope.saveInProgress = false; + $(element).modal('hide'); + }) + } else { + $http.post('/api/dashboards', { + 'name': $scope.dashboard.name + }).success(function(response) { + $(element).modal('hide'); + $location.path('/dashboard/' + response.slug).replace(); + }) + } + } + + } + } + } + ]); + + directives.directive('newWidgetForm', ['$http', 'Query', + function($http, Query) { + return { + restrict: 'E', + scope: { + dashboard: '=' + }, + templateUrl: '/views/new_widget_form.html', + replace: true, + link: function($scope, element, attrs) { + $scope.widgetSizes = [{ + name: 'Regular', + value: 1 + }, { + name: 'Double', + value: 2 + }]; + + var reset = function() { + $scope.saveInProgress = false; + $scope.widgetSize = 1; + $scope.queryId = null; + $scope.selectedVis = null; + $scope.query = null; + + } + + reset(); + + $scope.loadVisualizations = function() { + if (!$scope.queryId) { + return; + } + + Query.get({ + id: $scope.queryId + }, function(query) { + if (query) { + $scope.query = query; + if (query.visualizations.length) { + $scope.selectedVis = query.visualizations[0]; + } + } + }); + }; + + $scope.saveWidget = function() { + $scope.saveInProgress = true; + + var widget = { + 'visualization_id': $scope.selectedVis.id, + 'dashboard_id': $scope.dashboard.id, + 'options': {}, + 'width': $scope.widgetSize + } + + $http.post('/api/widgets', widget).success(function(response) { + // update dashboard layout + $scope.dashboard.layout = response['layout']; + if (response['new_row']) { + $scope.dashboard.widgets.push([response['widget']]); + } else { + $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']); + } + + // close the dialog + $('#add_query_dialog').modal('hide'); + reset(); + }) + } + + } + } + } + ]) +})(); \ No newline at end of file diff --git a/rd_ui/app/scripts/directives/directives.js b/rd_ui/app/scripts/directives/directives.js index b78246401..f4fc5f765 100644 --- a/rd_ui/app/scripts/directives/directives.js +++ b/rd_ui/app/scripts/directives/directives.js @@ -83,169 +83,6 @@ } }]); - directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) { - return { - restrict: 'E', - scope: { - dashboard: '=' - }, - templateUrl: '/views/edit_dashboard.html', - replace: true, - link: function($scope, element, attrs) { - var gridster = element.find(".gridster ul").gridster({ - widget_margins: [5, 5], - widget_base_dimensions: [260, 100], - min_cols: 2, - max_cols: 2, - serialize_params: function($w, wgd) { - return { - col: wgd.col, - row: wgd.row, - id: $w.data('widget-id') - } - } - }).data('gridster'); - - var gsItemTemplate = '
    • ' + - '
      {name}' + - '
      • '; - - $scope.$watch('dashboard.widgets', function(widgets) { - $timeout(function () { - gridster.remove_all_widgets(); - - if (widgets && widgets.length) { - var layout = []; - - _.each(widgets, function(row, rowIndex) { - _.each(row, function(widget, colIndex) { - layout.push({ - id: widget.id, - col: colIndex+1, - row: rowIndex+1, - ySize: 1, - xSize: widget.width, - name: widget.visualization.query.name - }); - }); - }); - - _.each(layout, function(item) { - var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); - gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); - - }); - } - }); - }, true); - - $scope.saveDashboard = function() { - $scope.saveInProgress = true; - // TODO: we should use the dashboard service here. - if ($scope.dashboard.id) { - var positions = $(element).find('.gridster ul').data('gridster').serialize(); - var layout = []; - _.each(_.sortBy(positions, function (pos) { - return pos.row * 10 + pos.col; - }), function (pos) { - var row = pos.row - 1; - var col = pos.col - 1; - layout[row] = layout[row] || []; - if (col > 0 && layout[row][col - 1] == undefined) { - layout[row][col - 1] = pos.id; - } else { - layout[row][col] = pos.id; - } - - }); - $scope.dashboard.layout = layout; - - layout = JSON.stringify(layout); - $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) { - $scope.dashboard = new Dashboard(response); - $scope.saveInProgress = false; - $(element).modal('hide'); - }) - } else { - $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) { - $(element).modal('hide'); - $location.path('/dashboard/' + response.slug).replace(); - }) - } - } - - } - } - }]); - - directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) { - return { - restrict: 'E', - scope: { - dashboard: '=' - }, - templateUrl: '/views/new_widget_form.html', - replace: true, - link: function($scope, element, attrs) { - $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}]; - - var reset = function() { - $scope.saveInProgress = false; - $scope.widgetSize = 1; - $scope.queryId = null; - $scope.selectedVis = null; - $scope.query = null; - - } - - reset(); - - $scope.loadVisualizations = function() { - if (!$scope.queryId) { - return; - } - - Query.get({ - id: $scope.queryId - }, function(query) { - if (query) { - $scope.query = query; - if(query.visualizations.length) { - $scope.selectedVis = query.visualizations[0]; - } - } - }); - }; - - $scope.saveWidget = function() { - $scope.saveInProgress = true; - - var widget = { - 'visualization_id': $scope.selectedVis.id, - 'dashboard_id': $scope.dashboard.id, - 'options': {}, - 'width': $scope.widgetSize - } - - $http.post('/api/widgets', widget).success(function(response) { - // update dashboard layout - $scope.dashboard.layout = response['layout']; - if (response['new_row']) { - $scope.dashboard.widgets.push([response['widget']]); - } else { - $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']); - } - - // close the dialog - $('#add_query_dialog').modal('hide'); - reset(); - }) - } - - } - } - }]) - // From: http://jsfiddle.net/joshdmiller/NDFHg/ directives.directive('editInPlace', function () { return { From 3243f277f256acec20992433d3fcb24293c6ce9d Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 13:25:21 +0200 Subject: [PATCH 456/540] [#90] query link: style, link to #table, 'query' attr required --- rd_ui/app/scripts/directives/query_directives.js | 13 ++++++++----- rd_ui/app/styles/redash.css | 3 ++- rd_ui/app/views/dashboard.html | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index ba06e7e36..4f7ef902e 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -5,16 +5,19 @@ return { restrict: 'E', scope: { - 'query': '=?', + 'query': '=', 'visualization': '=?' }, - template: '{{query.name}}', + template: '{{query.name}}', link: function(scope, element) { - scope.query = scope.query || scope.$parent.query; - var link = '/queries/' + scope.query.id; if (scope.visualization) { - link += '#' + scope.visualization.id; + if (scope.visualization.type === 'TABLE') { + // link to hard-coded table tab instead of the (hidden) visualization tab + link += '#table'; + } else { + link += '#' + scope.visualization.id; + } } element.find('a').attr('href', link); } diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index a5c7410d4..3a1a6e50a 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -74,7 +74,8 @@ a.navbar-brand { margin-bottom: 0px; } -.panel-heading > a { +.panel-heading > a, +.panel-heading .query-link { color: inherit; } diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 8d13b7447..a872ee2c8 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -25,7 +25,7 @@

      - +

      From fd3e9e3fcb962f022e5cb0ef6e9cf7fd1c3626b1 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 13:30:50 +0200 Subject: [PATCH 457/540] query links: no underline --- rd_ui/app/styles/redash.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 3a1a6e50a..c0078fd27 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -79,6 +79,10 @@ a.navbar-brand { color: inherit; } +.panel-heading .query-link:hover { + text-decoration: none; +} + /* angular-growl */ .growl { position: fixed; From ace657d95a0f89c6b75547ea0c9ec43dc2514941 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 16:42:56 +0200 Subject: [PATCH 458/540] [#90] handle widget creation failures --- .../scripts/directives/dashboard_directives.js | 15 +++++++++------ rd_ui/app/scripts/services/resources.js | 9 ++++++++- rd_ui/app/views/new_widget_form.html | 4 ++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js index ff707348a..44b3ccc50 100644 --- a/rd_ui/app/scripts/directives/dashboard_directives.js +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -105,8 +105,8 @@ } ]); - directives.directive('newWidgetForm', ['$http', 'Query', - function($http, Query) { + directives.directive('newWidgetForm', ['Query', 'Widget', 'growl', + function(Query, Widget, growl) { return { restrict: 'E', scope: { @@ -154,14 +154,14 @@ $scope.saveWidget = function() { $scope.saveInProgress = true; - var widget = { + var widget = new Widget({ 'visualization_id': $scope.selectedVis.id, 'dashboard_id': $scope.dashboard.id, 'options': {}, 'width': $scope.widgetSize - } + }); - $http.post('/api/widgets', widget).success(function(response) { + widget.$save(function success(response) { // update dashboard layout $scope.dashboard.layout = response['layout']; if (response['new_row']) { @@ -173,7 +173,10 @@ // close the dialog $('#add_query_dialog').modal('hide'); reset(); - }) + }, function error(httpResponse) { + $scope.saveInProgress = false; + growl.addErrorMessage("Widget can not be added"); + }); } } diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 7867de40e..61111e5df 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -339,8 +339,15 @@ return DataSourceResource; } + var Widget = function($resource) { + var WidgetResource = $resource('/api/widgets/:id', {id: '@id'}); + + return WidgetResource; + } + angular.module('redash.services') .factory('QueryResult', ['$resource', '$timeout', QueryResult]) .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query]) - .factory('DataSource', ['$resource', DataSource]); + .factory('DataSource', ['$resource', DataSource]) + .factory('Widget', ['$resource', Widget]); })(); diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 41a1e0299..10b41b43c 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -12,7 +12,7 @@

      @@ -29,7 +29,7 @@
      -
      +
      From d4d118af17c7cc5fc9f6a80f7318562a4b2f088e Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 17:27:18 +0200 Subject: [PATCH 459/540] update angular-resource (adds .$promise support) --- rd_ui/app/scripts/app.js | 17 +++++------------ .../scripts/directives/dashboard_directives.js | 7 ++++--- rd_ui/bower.json | 4 ++-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 9ed52fba5..fd0e4ab9c 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -17,16 +17,9 @@ angular.module('redash', [ ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', function($routeProvider, $locationProvider, $compileProvider, growlProvider) { - function getQuery(Query, $q, $route) { - var defer = $q.defer(); - - Query.get({ - 'id': $route.current.params.queryId - }, function(query) { - defer.resolve(query); - }); - - return defer.promise; + function getQuery(Query, $route) { + var query = Query.get({'id': $route.current.params.queryId }); + return query.$promise; }; $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/); @@ -57,7 +50,7 @@ angular.module('redash', [ controller: 'QueryViewCtrl', reloadOnSearch: false, resolve: { - 'query': ['Query', '$q', '$route', getQuery] + 'query': ['Query', '$route', getQuery] } }); $routeProvider.when('/queries/:queryId/source', { @@ -65,7 +58,7 @@ angular.module('redash', [ controller: 'QueryEditCtrl', reloadOnSearch: false, resolve: { - 'query': ['Query', '$q', '$route', getQuery] + 'query': ['Query', '$route', getQuery] } }); $routeProvider.when('/admin/status', { diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js index 44b3ccc50..6142271c3 100644 --- a/rd_ui/app/scripts/directives/dashboard_directives.js +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -161,7 +161,7 @@ 'width': $scope.widgetSize }); - widget.$save(function success(response) { + widget.$save().then(function(response) { // update dashboard layout $scope.dashboard.layout = response['layout']; if (response['new_row']) { @@ -173,9 +173,10 @@ // close the dialog $('#add_query_dialog').modal('hide'); reset(); - }, function error(httpResponse) { - $scope.saveInProgress = false; + }).catch(function() { growl.addErrorMessage("Widget can not be added"); + }).finally(function() { + $scope.saveInProgress = false; }); } diff --git a/rd_ui/bower.json b/rd_ui/bower.json index a6954a8a0..d8bb06a0b 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -13,7 +13,7 @@ "angular-ui-codemirror": "0.0.5", "highcharts": "3.0.1", "underscore": "1.5.1", - "angular-resource": "1.2.7", + "angular-resource": "1.2.15", "angular-growl": "0.3.1", "angular-route": "1.2.7", "pivottable": "https://github.com/arikfr/pivottable.git", @@ -26,6 +26,6 @@ "angular-scenario": "~1.0.7" }, "resolutions": { - "angular": "~1.2.7" + "angular": "1.2.7" } } From 40adba4242e81905d91359aa05c4fe867c875f97 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Wed, 26 Mar 2014 17:52:07 +0200 Subject: [PATCH 460/540] [#90] query-link: use ng-href as @arikfr suggested --- rd_ui/app/scripts/directives/query_directives.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 4f7ef902e..b8f29eec9 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -8,18 +8,18 @@ 'query': '=', 'visualization': '=?' }, - template: '{{query.name}}', + template: '{{query.name}}', link: function(scope, element) { - var link = '/queries/' + scope.query.id; + scope.link = '/queries/' + scope.query.id; if (scope.visualization) { if (scope.visualization.type === 'TABLE') { // link to hard-coded table tab instead of the (hidden) visualization tab - link += '#table'; + scope.link += '#table'; } else { - link += '#' + scope.visualization.id; + scope.link += '#' + scope.visualization.id; } } - element.find('a').attr('href', link); + // element.find('a').attr('href', link); } } } From b2e7813d87782ba2f37956e6cbb200c678015666 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 30 Mar 2014 17:30:32 +0300 Subject: [PATCH 461/540] Fix: some dashboards get into infinite loop of watches --- rd_ui/app/scripts/directives/dashboard_directives.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js index 6142271c3..d39ae2c37 100644 --- a/rd_ui/app/scripts/directives/dashboard_directives.js +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -31,14 +31,14 @@ '
      {name}' + '
      '; - $scope.$watch('dashboard.widgets', function(widgets) { + $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) { $timeout(function() { gridster.remove_all_widgets(); - if (widgets && widgets.length) { + if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) { var layout = []; - _.each(widgets, function(row, rowIndex) { + _.each($scope.dashboard.widgets, function(row, rowIndex) { _.each(row, function(widget, colIndex) { layout.push({ id: widget.id, @@ -58,7 +58,7 @@ }); } }); - }, true); + }); $scope.saveDashboard = function() { $scope.saveInProgress = true; From 29f01a5780f69c43eb1c89cfff040efe8154a31d Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 27 Mar 2014 15:03:51 +0200 Subject: [PATCH 462/540] [#90] clear visualization hash when redirecting to forked query --- rd_ui/app/scripts/controllers/query_edit.js | 35 ++++++++++++++-- rd_ui/app/scripts/controllers/query_view.js | 45 +++++++-------------- rd_ui/app/scripts/services/resources.js | 4 ++ 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_edit.js index 9d8fbc87b..bf67ac03d 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_edit.js @@ -8,6 +8,10 @@ var isNewQuery = !$scope.query.id, queryText = $scope.query.query, + + // ref to QueryViewCtrl.saveQuery + saveQuery = $scope.saveQuery, + shortcuts = { 'meta+s': function() { if ($scope.canEdit) { @@ -24,9 +28,34 @@ KeyboardShortcuts.bind(shortcuts); - $scope.onQuerySave = function(savedQuery) { - queryText = savedQuery.query; - $scope.isDirty = $scope.query.query !== queryText; + // @override + $scope.saveQuery = function(options, data) { + var savePromise = saveQuery(options, data); + + savePromise.then(function(savedQuery) { + queryText = savedQuery.query; + $scope.isDirty = $scope.query.query !== queryText; + + if (isNewQuery) { + // redirect to new created query (keep hash) + $location.path(savedQuery.getSourceLink()).replace(); + } + }); + + return savePromise; + }; + + $scope.duplicateQuery = function() { + $scope.query.id = null; + $scope.query.ttl = -1; + + $scope.saveQuery({ + successMessage: 'Query forked', + errorMessage: 'Query could not be forked' + }).then(function redirect(savedQuery) { + // redirect to forked query (clear hash) + $location.url(savedQuery.getSourceLink()).replace() + }); }; $scope.deleteVisualization = function($e, vis) { diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 8a757ce23..464e42fb6 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -15,55 +15,38 @@ $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id; }); - $scope.onQuerySave = angular.noop; - $scope.lockButton = function(lock) { $scope.queryExecuting = lock; }; - $scope.saveQuery = function(duplicate, data) { - var - successMessage = "Query saved", - oldId = $scope.query.id; - - if (duplicate) { - successMessage = "Query forked"; - $scope.query.id = null; - $scope.query.ttl = -1; - } - + $scope.saveQuery = function(options, data) { if (data) { data.id = $scope.query.id; } else { data = $scope.query; } + options = _.extend({}, { + successMessage: 'Query saved', + errorMessage: 'Query could not be saved' + }, options); + delete $scope.query.latest_query_data; - Query.save(data, function(savedQuery) { - $scope.onQuerySave(savedQuery); - - if (oldId != savedQuery.id) { - // redirect to new/duplicated query page - $location.url('/queries/' + savedQuery.id + '/source#' + $location.hash()).replace(); - } - - growl.addSuccessMessage(successMessage); + return Query.save(data, function() { + growl.addSuccessMessage(options.successMessage); }, function(httpResponse) { - growl.addErrorMessage("Query could not be saved"); - }); - }; + growl.addErrorMessage(options.errorMessage); + }) + .$promise; + } $scope.saveDescription = function() { - $scope.saveQuery(false, {'description': $scope.query.description}); + $scope.saveQuery(undefined, {'description': $scope.query.description}); }; $scope.saveName = function() { - $scope.saveQuery(false, {'name': $scope.query.name}); - }; - - $scope.duplicateQuery = function() { - $scope.saveQuery(true); + $scope.saveQuery(undefined, {'name': $scope.query.name}); }; $scope.executeQuery = function() { diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 61111e5df..d2ceeed0c 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -313,6 +313,10 @@ }); }; + Query.prototype.getSourceLink = function() { + return '/queries/' + this.id + '/source'; + }; + Query.prototype.getQueryResult = function(ttl) { if (ttl == undefined) { ttl = this.ttl; From 87933bd8acee73b456f7325de8b267ab14fe6f8c Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 27 Mar 2014 15:07:47 +0200 Subject: [PATCH 463/540] rename: QueryEditCtrl -> QuerySourceCtrl 'edit' is confusing since it is also possible to make changes in the QueryViewCtrl --- rd_ui/app/index.html | 2 +- rd_ui/app/scripts/app.js | 4 ++-- .../scripts/controllers/{query_edit.js => query_source.js} | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) rename rd_ui/app/scripts/controllers/{query_edit.js => query_source.js} (91%) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index c300dfcd1..1a74f80f4 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -123,7 +123,7 @@ - + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index fd0e4ab9c..75594fb15 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -37,7 +37,7 @@ angular.module('redash', [ }); $routeProvider.when('/queries/new', { templateUrl: '/views/query.html', - controller: 'QueryEditCtrl', + controller: 'QuerySourceCtrl', reloadOnSearch: false, resolve: { 'query': ['Query', function newQuery(Query) { @@ -55,7 +55,7 @@ angular.module('redash', [ }); $routeProvider.when('/queries/:queryId/source', { templateUrl: '/views/query.html', - controller: 'QueryEditCtrl', + controller: 'QuerySourceCtrl', reloadOnSearch: false, resolve: { 'query': ['Query', '$route', getQuery] diff --git a/rd_ui/app/scripts/controllers/query_edit.js b/rd_ui/app/scripts/controllers/query_source.js similarity index 91% rename from rd_ui/app/scripts/controllers/query_edit.js rename to rd_ui/app/scripts/controllers/query_source.js index bf67ac03d..99d4f2b60 100644 --- a/rd_ui/app/scripts/controllers/query_edit.js +++ b/rd_ui/app/scripts/controllers/query_source.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - function QueryEditCtrl($controller, $scope, $location, growl, Query, Visualization, KeyboardShortcuts) { + function QuerySourceCtrl($controller, $scope, $location, growl, Query, Visualization, KeyboardShortcuts) { // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); @@ -93,8 +93,8 @@ }; - angular.module('redash.controllers').controller('QueryEditCtrl', [ + angular.module('redash.controllers').controller('QuerySourceCtrl', [ '$controller', '$scope', '$location', 'growl', 'Query', - 'Visualization', 'KeyboardShortcuts', QueryEditCtrl + 'Visualization', 'KeyboardShortcuts', QuerySourceCtrl ]); })(); \ No newline at end of file From a696e10ef7a526d55b49b2b52d869c1f662ac1d9 Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 27 Mar 2014 17:53:03 +0200 Subject: [PATCH 464/540] [#90] visualization edit mode --- rd_ui/app/styles/redash.css | 12 ++++++++++++ rd_ui/app/views/query.html | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index c0078fd27..facbff214 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -37,6 +37,18 @@ a.navbar-brand { bottom: -11px; } +.details-toggle { + cursor: pointer; +} +.details-toggle::before { + content: 'â–¸'; + margin-right: 5px; +} +.details-toggle.open::before { + content: 'â–¾'; + margin-right: 5px; +} + .edit-in-place span { white-space: pre-line; } diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 113a38f8f..370a8c2db 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -165,8 +165,11 @@
      + Edit + + - +
      From 82f5f15c2a706e7d48963e57d10ddbee97d9065f Mon Sep 17 00:00:00 2001 From: Amir Nissim Date: Thu, 3 Apr 2014 15:14:27 +0300 Subject: [PATCH 465/540] [#90] edit vis. form touchup --- rd_ui/app/views/query.html | 4 +- .../views/visualizations/chart_editor.html | 14 ++++--- .../visualizations/edit_visualization.html | 40 ++++++++++--------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 370a8c2db..926a9ff2b 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -165,11 +165,9 @@
      - Edit - - +
      diff --git a/rd_ui/app/views/visualizations/chart_editor.html b/rd_ui/app/views/visualizations/chart_editor.html index 2c36235a4..71dee8ea0 100644 --- a/rd_ui/app/views/visualizations/chart_editor.html +++ b/rd_ui/app/views/visualizations/chart_editor.html @@ -1,7 +1,11 @@ -
      - - +
      +
      + + +
      - - +
      + + +
      \ No newline at end of file diff --git a/rd_ui/app/views/visualizations/edit_visualization.html b/rd_ui/app/views/visualizations/edit_visualization.html index 7911aef28..32bace4ab 100644 --- a/rd_ui/app/views/visualizations/edit_visualization.html +++ b/rd_ui/app/views/visualizations/edit_visualization.html @@ -1,23 +1,27 @@ -
      -
      - - -
      +
      + Edit -
      - - -
      + +
      + + +
      - +
      + + +
      -
      - - -
      + -
      - -
      +
      + + +
      - +
      + +
      + + +
      \ No newline at end of file From c0fc7c82224679e3fbe8496b857f56c4f449f632 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 19:16:10 +0300 Subject: [PATCH 466/540] new permission: view_query --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index 8fd484b57..ba196ee5c 100644 --- a/redash/models.py +++ b/redash/models.py @@ -24,7 +24,7 @@ class AnonymousUser(AnonymousUserMixin): class User(BaseModel, UserMixin): DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query', - 'view_source', 'execute_query'] + 'view_query', 'view_source', 'execute_query'] id = peewee.PrimaryKeyField() name = peewee.CharField(max_length=320) From 3e6dd8e929a5dbb75d6a061cdadc8d145fd8d53d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 19:16:18 +0300 Subject: [PATCH 467/540] Migration for new permission --- migrations/add_view_query_permission.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 migrations/add_view_query_permission.py diff --git a/migrations/add_view_query_permission.py b/migrations/add_view_query_permission.py new file mode 100644 index 000000000..7eb9ccc83 --- /dev/null +++ b/migrations/add_view_query_permission.py @@ -0,0 +1,13 @@ +import peewee +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + + previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:] + previous_default_permissions.remove('view_query') + models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute() + + db.close_db(None) From 99a9fdde25718f5080ee7f773fb17b3c4e11fdca Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 19:16:30 +0300 Subject: [PATCH 468/540] Use view_query permission in controllers --- redash/controllers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/redash/controllers.py b/redash/controllers.py index eefd57168..7ebde0a84 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -249,6 +249,7 @@ class QueryListAPI(BaseResource): return query.to_dict(with_result=False) + @require_permission('view_query') def get(self): return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()] @@ -272,6 +273,7 @@ class QueryAPI(BaseResource): return query.to_dict(with_result=False, with_visualizations=True) + @require_permission('view_query') def get(self, query_id): q = models.Query.get(models.Query.id == query_id) if q: @@ -345,6 +347,7 @@ class QueryResultListAPI(BaseResource): class QueryResultAPI(BaseResource): + @require_permission('view_query') def get(self, query_result_id): query_result = models.QueryResult.get_by_id(query_result_id) if query_result: @@ -354,6 +357,7 @@ class QueryResultAPI(BaseResource): class CsvQueryResultsAPI(BaseResource): + @require_permission('view_query') def get(self, query_id, query_result_id=None): if not query_result_id: query = models.Query.get(models.Query.id == query_id) From 5de1795380df93c48bc5b4e65b4d761e75fda747 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 19:32:46 +0300 Subject: [PATCH 469/540] Don't show links to queries in the UI. --- rd_ui/app/index.html | 2 +- rd_ui/app/views/dashboard.html | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index c300dfcd1..aaf4bf1f2 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -55,7 +55,7 @@
    • New Dashboard
      • -
    • +
    • Queries
      • New Query
        • diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index a872ee2c8..d881680ac 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -25,7 +25,8 @@

        - + {{query.name}} +

        @@ -39,7 +40,7 @@ tooltip-placement="bottom">Updated: - + From 0167bebf042b182e4f321680b92ccb249d41c9cf Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 20:05:43 +0300 Subject: [PATCH 470/540] Create stub User object for API to use permissions model --- redash/authentication.py | 1 + redash/models.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/redash/authentication.py b/redash/authentication.py index f2b044703..fe3258f97 100644 --- a/redash/authentication.py +++ b/redash/authentication.py @@ -40,6 +40,7 @@ class HMACAuthentication(object): calculated_signature = sign(query.api_key, request.path, expires) if query.api_key and signature == calculated_signature: + login_user(models.ApiUser(query.api_key), remember=False) return True return False diff --git a/redash/models.py b/redash/models.py index ba196ee5c..44c313819 100644 --- a/redash/models.py +++ b/redash/models.py @@ -22,6 +22,15 @@ class AnonymousUser(AnonymousUserMixin): return [] +class ApiUser(UserMixin): + def __init__(self, api_key): + self.id = api_key + + @property + def permissions(self): + return ['view_query'] + + class User(BaseModel, UserMixin): DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query', 'view_query', 'view_source', 'execute_query'] From 81ca8b901277e4f49c0ffc0f40d904e8bdcfb6ee Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 6 Apr 2014 20:26:35 +0300 Subject: [PATCH 471/540] More control over creating users from CLI --- manage.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/manage.py b/manage.py index a9c91c61f..a00bdad3c 100755 --- a/manage.py +++ b/manage.py @@ -80,18 +80,22 @@ def drop_tables(): @users_manager.option('name', help="User's full name") @users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin") @users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login") -def create(email, name, is_admin=False, google_auth=False): +@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).") +@users_manager.option('--permissions', dest='permissions', default=models.User.DEFAULT_PERMISSIONS, help="Comma seperated list of permissions (leave blank for default).") +def create(email, name, permissions, is_admin=False, google_auth=False, password=None): print "Creating user (%s, %s)..." % (email, name) print "Admin: %r" % is_admin print "Login with Google Auth: %r\n" % google_auth + if isinstance(permissions, basestring): + permissions = permissions.split(',') + permissions.remove('') # in case it was empty string - permissions = models.User.DEFAULT_PERMISSIONS if is_admin: permissions += ['admin'] user = models.User(email=email, name=name, permissions=permissions) if not google_auth: - password = prompt_pass("Password") + password = password or prompt_pass("Password") user.hash_password(password) try: From 2fd3033418744021b60ca234cdcbc8353592e461 Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 7 Apr 2014 12:14:10 +0300 Subject: [PATCH 472/540] Make sure qr serialization will always be in JSON - in the case we do end up serializing big objects - so that other parts of the system can be written in languages other than Python --- redash/data/manager.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) mode change 100644 => 100755 redash/data/manager.py diff --git a/redash/data/manager.py b/redash/data/manager.py old mode 100644 new mode 100755 index c5edcdb82..3b63a9cfe --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -6,17 +6,25 @@ import logging import peewee import qr import redis +import json from redash import models from redash.data import worker from redash.utils import gen_query_hash +class JSONPriorityQueue(qr.PriorityQueue): + """ Use a JSON serializer to help with cross language support """ + def __init__(self, key, **kwargs): + super(qr.PriorityQueue, self).__init__(key, **kwargs) + self.serializer = json + + class Manager(object): def __init__(self, redis_connection, statsd_client): self.statsd_client = statsd_client self.redis_connection = redis_connection self.workers = [] - self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) + self.queue = JSONPriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs) self.max_retries = 5 self.status = { 'last_refresh_at': 0, From a37142426cb9339cca21865d34e5af77cf058b96 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 7 Apr 2014 18:17:12 +0300 Subject: [PATCH 473/540] Fix: when deleting visualization it would fail because DEFAULT_TAB is undefined --- rd_ui/app/scripts/controllers/query_source.js | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_source.js b/rd_ui/app/scripts/controllers/query_source.js index 99d4f2b60..f973ee84b 100644 --- a/rd_ui/app/scripts/controllers/query_source.js +++ b/rd_ui/app/scripts/controllers/query_source.js @@ -1,24 +1,25 @@ (function() { 'use strict'; - function QuerySourceCtrl($controller, $scope, $location, growl, Query, Visualization, KeyboardShortcuts) { + function QuerySourceCtrl($controller, $scope, $location, Query, Visualization, KeyboardShortcuts) { // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); + // TODO: + // This doesn't get inherited. Setting it on this didn't work either (which is weird). + // Obviously it shouldn't be repeated, but we got bigger fish to fry. + var DEFAULT_TAB = 'table'; - var - isNewQuery = !$scope.query.id, - queryText = $scope.query.query, - - // ref to QueryViewCtrl.saveQuery - saveQuery = $scope.saveQuery, - - shortcuts = { - 'meta+s': function() { - if ($scope.canEdit) { - $scope.saveQuery(); + var isNewQuery = !$scope.query.id, + queryText = $scope.query.query, + // ref to QueryViewCtrl.saveQuery + saveQuery = $scope.saveQuery, + shortcuts = { + 'meta+s': function () { + if ($scope.canEdit) { + $scope.saveQuery(); + } } - } - }; + }; $scope.sourceMode = true; $scope.canEdit = currentUser.canEdit($scope.query); @@ -90,11 +91,10 @@ } }); } - - }; + } angular.module('redash.controllers').controller('QuerySourceCtrl', [ - '$controller', '$scope', '$location', 'growl', 'Query', + '$controller', '$scope', '$location', 'Query', 'Visualization', 'KeyboardShortcuts', QuerySourceCtrl ]); })(); \ No newline at end of file From b242295de03faf357caccfaf5b285ad884da454a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 7 Apr 2014 18:39:20 +0300 Subject: [PATCH 474/540] Feature: Control over xAxis type. --- rd_ui/app/scripts/ng_highchart.js | 547 +++++++++--------- rd_ui/app/scripts/visualizations/chart.js | 209 ++++--- .../views/visualizations/chart_editor.html | 3 + 3 files changed, 395 insertions(+), 364 deletions(-) diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index a17de6802..ee5e545c5 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -1,279 +1,290 @@ (function () { - 'use strict'; + 'use strict'; - Highcharts.setOptions({ - colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", - "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"] - }); + Highcharts.setOptions({ + colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE", + "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"] + }); - var defaultOptions = { - title: { - "text": null - }, - xAxis: { - type: 'datetime' - }, - yAxis: { - title: { - text: null + var defaultOptions = { + title: { + "text": null + }, + xAxis: { + type: 'datetime' + }, + yAxis: { + title: { + text: null + } + }, + tooltip: { + valueDecimals: 2, + formatter: function () { + if (!this.points) { + this.points = [this.point]; + } + ; + + if (moment.isMoment(this.x)) { + var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', + pointsCount = this.points.length; + + $.each(this.points, function (i, point) { + s += '
        ' + point.series.name + ': ' + + Highcharts.numberFormat(point.y); + + if (pointsCount > 1 && point.percentage) { + s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; } + }); + } else { + var points = this.points; + var name = points[0].key || points[0].name; + + var s = "" + name + ""; + + $.each(points, function (i, point) { + if (points.length > 1) { + s += '
        ' + point.series.name + ': ' + Highcharts.numberFormat(point.y); + } else { + s += ": " + Highcharts.numberFormat(point.y); + if (point.percentage < 100) { + s += ' (' + Highcharts.numberFormat(point.percentage) + '%)'; + } + } + }); + } + + return s; + }, + shared: true + }, + exporting: { + chartOptions: { + title: { + text: '' + } + }, + buttons: { + contextButton: { + menuItems: [ + { + text: 'Toggle % Stacking', + onclick: function () { + var newStacking = "normal"; + if (this.series[0].options.stacking == "normal") { + newStacking = "percent"; + } + + _.each(this.series, function (series) { + series.update({stacking: newStacking}, true); + }); + } + } + ] + } + } + }, + credits: { + enabled: false + }, + plotOptions: { + area: { + marker: { + enabled: false, + symbol: 'circle', + radius: 2, + states: { + hover: { + enabled: true + } + } + } + }, + column: { + stacking: "normal", + pointPadding: 0, + borderWidth: 1, + groupPadding: 0, + shadow: false + }, + line: { + marker: { + radius: 1 + }, + lineWidth: 2, + states: { + hover: { + lineWidth: 2, + marker: { + radius: 3 + } + } + } + }, + pie: { + allowPointSelect: true, + cursor: 'pointer', + dataLabels: { + enabled: true, + color: '#000000', + connectorColor: '#000000', + format: '{point.name}: {point.y} ({point.percentage:.1f} %)' + } + }, + scatter: { + marker: { + radius: 5, + states: { + hover: { + enabled: true, + lineColor: 'rgb(100,100,100)' + } + } }, tooltip: { - valueDecimals: 2, - formatter: function () { - if (!this.points) { - this.points = [this.point]; - }; + headerFormat: '{series.name}
        ', + pointFormat: '{point.x}, {point.y}' + } + } + }, + series: [] + }; - if (moment.isMoment(this.x)) { - var s = '' + moment(this.x).format("DD/MM/YY HH:mm") + '', - pointsCount = this.points.length; + angular.module('highchart', []) + .directive('chart', ['$timeout', function ($timeout) { + return { + restrict: 'E', + template: '
        ', + scope: { + options: "=options", + series: "=series" + }, + transclude: true, + replace: true, - $.each(this.points, function (i, point) { - s += '
        ' + point.series.name + ': ' + - Highcharts.numberFormat(point.y); - - if (pointsCount > 1 && point.percentage) { - s += " (" + Highcharts.numberFormat(point.percentage) + "%)"; - } - }); - } else { - var points = this.points; - var name = points[0].key || points[0].name; - - var s = "" + name + ""; - - $.each(points, function (i, point) { - if (points.length > 1) { - s += '
        ' + point.series.name + ': ' + Highcharts.numberFormat(point.y); - } else { - s += ": " + Highcharts.numberFormat(point.y); - if (point.percentage < 100) { - s += ' (' +Highcharts.numberFormat(point.percentage) + '%)'; - } - } - }); - } - - return s; - }, - shared: true - }, - exporting: { - chartOptions: { - title: { - text: '' - } - }, - buttons: { - contextButton: { - menuItems: [ - { - text: 'Toggle % Stacking', - onclick: function () { - var newStacking = "normal"; - if (this.series[0].options.stacking == "normal") { - newStacking = "percent"; - } - - _.each(this.series, function (series) { - series.update({stacking: newStacking}, true); - }); - } - } - ] - } - } - }, - credits: { - enabled: false - }, - plotOptions: { - area: { - marker: { - enabled: false, - symbol: 'circle', - radius: 2, - states: { - hover: { - enabled: true - } - } - } - }, - column: { - stacking: "normal", - pointPadding: 0, - borderWidth: 1, - groupPadding: 0, - shadow: false - }, - line: { - marker: { - radius: 1 - }, - lineWidth: 2, - states: { - hover: { - lineWidth: 2, - marker: { - radius: 3 - } - } - } - }, - pie: { - allowPointSelect: true, - cursor: 'pointer', - dataLabels: { - enabled: true, - color: '#000000', - connectorColor: '#000000', - format: '{point.name}: {point.y} ({point.percentage:.1f} %)' - } - }, - scatter: { - marker: { - radius: 5, - states: { - hover: { - enabled: true, - lineColor: 'rgb(100,100,100)' - } - } - }, - tooltip: { - headerFormat: '{series.name}
        ', - pointFormat: '{point.x}, {point.y}' - } - } - }, - series: [] - }; - - angular.module('highchart', []) - .directive('chart', ['$timeout', function ($timeout) { - return { - restrict: 'E', - template: '
        ', - scope: { - options: "=options", - series: "=series" - }, - transclude: true, - replace: true, - - link: function (scope, element, attrs) { - var chartsDefaults = { - chart: { - renderTo: element[0], - type: attrs.type || null, - height: attrs.height || null, - width: attrs.width || null - } - }; - - var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults); - - // $timeout makes sure that this function invoked after the DOM ready. When draw/init - // invoked after the DOM is ready, we see first an empty HighCharts objects and later - // they get filled up. Which gives the feeling that the charts loading faster (otherwise - // we stare at an empty screen until the HighCharts object is ready). - $timeout(function(){ - // Update when options change - scope.$watch('options', function (newOptions) { - initChart(newOptions); - }, true); - - //Update when charts data changes - scope.$watchCollection('series', function (series) { - if (!series || series.length == 0) { - scope.chart.showLoading(); - } else { - drawChart(); - }; - }); - }); - - function initChart(options) { - if (scope.chart) { - scope.chart.destroy(); - }; - - $.extend(true, chartOptions, options); - - scope.chart = new Highcharts.Chart(chartOptions); - drawChart(); - } - - function drawChart() { - while (scope.chart.series.length > 0) { - scope.chart.series[0].remove(false); - }; - - if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) { - return (angular.isString(p.x) || angular.isDefined(p.name)); - })) { - scope.chart.xAxis[0].update({type: 'category'}); - - if (!angular.isDefined(scope.series[0].data[0].name)) { - // We need to make sure that for each category, each series has a value. - var categories = _.union.apply(this, _.map(scope.series, function (s) { - return _.pluck(s.data, 'x') - })); - - _.each(scope.series, function (s) { - // TODO: move this logic to Query#getChartData - var yValues = _.groupBy(s.data, 'x'); - - var newData = _.map(categories, function (category) { - return { - name: category, - y: (yValues[category] && yValues[category][0].y) || 0 - } - }); - - if (categories.length == 1) { - newData = _.sortBy(newData, 'y').reverse(); - }; - - s.data = newData; - }); - } - } else { - scope.chart.xAxis[0].update({type: 'datetime'}); - } - - scope.chart.counters.color = 0; - - _.each(scope.series, function (s) { - // here we override the series with the visualization config - s = _.extend(s, chartOptions['series']); - - if (s.type == 'area') { - _.each(s.data, function (p) { - // This is an insane hack: somewhere deep in HighChart's code, - // when you stack areas, it tries to convert the string representation - // of point's x into a number. With the default implementation of toString - // it fails.... - - if (moment.isMoment(p.x)) { - p.x.toString = function () { - return String(this.toDate().getTime()); - }; - } - }); - }; - - scope.chart.addSeries(s, false); - }); - - scope.chart.redraw(); - scope.chart.hideLoading(); - } - - } + link: function (scope, element, attrs) { + var chartsDefaults = { + chart: { + renderTo: element[0], + type: attrs.type || null, + height: attrs.height || null, + width: attrs.width || null + } }; - }]); + var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults); + + // $timeout makes sure that this function invoked after the DOM ready. When draw/init + // invoked after the DOM is ready, we see first an empty HighCharts objects and later + // they get filled up. Which gives the feeling that the charts loading faster (otherwise + // we stare at an empty screen until the HighCharts object is ready). + $timeout(function () { + // Update when options change + scope.$watch('options', function (newOptions) { + initChart(newOptions); + }, true); + + //Update when charts data changes + scope.$watchCollection('series', function (series) { + if (!series || series.length == 0) { + scope.chart.showLoading(); + } else { + drawChart(); + } + ; + }); + }); + + function initChart(options) { + if (scope.chart) { + scope.chart.destroy(); + } + ; + + $.extend(true, chartOptions, options); + + scope.chart = new Highcharts.Chart(chartOptions); + drawChart(); + } + + function drawChart() { + while (scope.chart.series.length > 0) { + scope.chart.series[0].remove(false); + }; + + if (!('xAxis' in chartOptions && 'type' in chartOptions['xAxis'])) { + if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) { + return (angular.isString(p.x) || angular.isDefined(p.name)); + })) { + chartOptions['xAxis'] = chartOptions['xAxis'] || {}; + chartOptions['xAxis']['type'] = 'category'; + } else { + chartOptions['xAxis'] = chartOptions['xAxis'] || {}; + chartOptions['xAxis']['type'] = 'datetime'; + } + } + + if (chartOptions['xAxis']['type'] == 'category') { + if (!angular.isDefined(scope.series[0].data[0].name)) { + // We need to make sure that for each category, each series has a value. + var categories = _.union.apply(this, _.map(scope.series, function (s) { + return _.pluck(s.data, 'x') + })); + + _.each(scope.series, function (s) { + // TODO: move this logic to Query#getChartData + var yValues = _.groupBy(s.data, 'x'); + + var newData = _.map(categories, function (category) { + return { + name: category, + y: (yValues[category] && yValues[category][0].y) || 0 + } + }); + + if (categories.length == 1) { + newData = _.sortBy(newData, 'y').reverse(); + } + ; + + s.data = newData; + }); + } + } + + scope.chart.counters.color = 0; + + _.each(scope.series, function (s) { + // here we override the series with the visualization config + s = _.extend(s, chartOptions['series']); + + if (s.type == 'area') { + _.each(s.data, function (p) { + // This is an insane hack: somewhere deep in HighChart's code, + // when you stack areas, it tries to convert the string representation + // of point's x into a number. With the default implementation of toString + // it fails.... + + if (moment.isMoment(p.x)) { + p.x.toString = function () { + return String(this.toDate().getTime()); + }; + } + }); + } + ; + + scope.chart.addSeries(s, false); + }); + + scope.chart.redraw(); + scope.chart.hideLoading(); + } + + } + }; + + }]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js index 60ef89408..e0208d358 100644 --- a/rd_ui/app/scripts/visualizations/chart.js +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -1,106 +1,123 @@ (function () { - var chartVisualization = angular.module('redash.visualization'); + var chartVisualization = angular.module('redash.visualization'); - chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) { - var renderTemplate = ''; - var editTemplate = ''; - var defaultOptions = { - 'series': { - 'type': 'column', - 'stacking': null - } + chartVisualization.config(['VisualizationProvider', function (VisualizationProvider) { + var renderTemplate = ''; + var editTemplate = ''; + var defaultOptions = { + 'series': { + 'type': 'column', + 'stacking': null + } + }; + + VisualizationProvider.registerVisualization({ + type: 'CHART', + name: 'Chart', + renderTemplate: renderTemplate, + editorTemplate: editTemplate, + defaultOptions: defaultOptions + }); + }]); + + chartVisualization.directive('chartRenderer', function () { + return { + restrict: 'E', + scope: { + queryResult: '=', + options: '=?' + }, + template: "", + replace: false, + controller: ['$scope', function ($scope) { + $scope.chartSeries = []; + $scope.chartOptions = {}; + + $scope.$watch('options', function (chartOptions) { + if (chartOptions) { + $scope.chartOptions = chartOptions; + } + }); + $scope.$watch('queryResult && queryResult.getData()', function (data) { + if (!data || $scope.queryResult.getData() == null) { + $scope.chartSeries.splice(0, $scope.chartSeries.length); + } else { + $scope.chartSeries.splice(0, $scope.chartSeries.length); + + _.each($scope.queryResult.getChartData(), function (s) { + $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'})); + }); + } + }); + }] + } + }); + + chartVisualization.directive('chartEditor', function () { + return { + restrict: 'E', + templateUrl: '/views/visualizations/chart_editor.html', + link: function (scope, element, attrs) { + scope.seriesTypes = { + 'Line': 'line', + 'Column': 'column', + 'Area': 'area', + 'Scatter': 'scatter', + 'Pie': 'pie' }; - VisualizationProvider.registerVisualization({ - type: 'CHART', - name: 'Chart', - renderTemplate: renderTemplate, - editorTemplate: editTemplate, - defaultOptions: defaultOptions - }); - }]); + scope.stackingOptions = { + "None": "none", + "Normal": "normal", + "Percent": "percent" + }; - chartVisualization.directive('chartRenderer', function () { - return { - restrict: 'E', - scope: { - queryResult: '=', - options: '=?' - }, - template: "", - replace: false, - controller: ['$scope', function ($scope) { - $scope.chartSeries = []; - $scope.chartOptions = {}; + scope.xAxisOptions = { + "Date/Time": "datetime", + "Linear": "linear", + "Category": "category" + }; - $scope.$watch('options', function(chartOptions) { - if (chartOptions) { - $scope.chartOptions = chartOptions; - } - }); - $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data || $scope.queryResult.getData() == null) { - $scope.chartSeries.splice(0, $scope.chartSeries.length); - } else { - $scope.chartSeries.splice(0, $scope.chartSeries.length); + scope.xAxisType = "datetime"; + scope.stacking = "none"; - _.each($scope.queryResult.getChartData(), function (s) { - $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'})); - }); - } - }); - }] - } - }); + var chartOptionsUnwatch = null; - chartVisualization.directive('chartEditor', function () { - return { - restrict: 'E', - templateUrl: '/views/visualizations/chart_editor.html', - link: function (scope, element, attrs) { - scope.seriesTypes = { - 'Line': 'line', - 'Column': 'column', - 'Area': 'area', - 'Scatter': 'scatter', - 'Pie': 'pie' - }; - - scope.stackingOptions = { - "None": "none", - "Normal": "normal", - "Percent": "percent" - }; - - scope.stacking = "none"; - - var chartOptionsUnwatch = null; - - scope.$watch('visualization', function (visualization) { - if (visualization && visualization.type == 'CHART') { - if (scope.visualization.options.series.stacking === null) { - scope.stacking = "none"; - } else if (scope.visualization.options.series.stacking === undefined) { - scope.stacking = "normal"; - } else { - scope.stacking = scope.visualization.options.series.stacking; - } - - chartOptionsUnwatch = scope.$watch("stacking", function (stacking) { - if (stacking == "none") { - scope.visualization.options.series.stacking = null; - } else { - scope.visualization.options.series.stacking = stacking; - } - }); - } else { - if (chartOptionsUnwatch) { - chartOptionsUnwatch(); - chartOptionsUnwatch = null; - } - } - }); + scope.$watch('visualization', function (visualization) { + if (visualization && visualization.type == 'CHART') { + if (scope.visualization.options.series.stacking === null) { + scope.stacking = "none"; + } else if (scope.visualization.options.series.stacking === undefined) { + scope.stacking = "normal"; + } else { + scope.stacking = scope.visualization.options.series.stacking; } - } - }); + + chartOptionsUnwatch = scope.$watch("stacking", function (stacking) { + if (stacking == "none") { + scope.visualization.options.series.stacking = null; + } else { + scope.visualization.options.series.stacking = stacking; + } + }); + + xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) { + scope.visualization.options.xAxis = scope.visualization.options.xAxis || {}; + scope.visualization.options.xAxis.type = xAxisType; + }); + } else { + if (chartOptionsUnwatch) { + chartOptionsUnwatch(); + chartOptionsUnwatch = null; + } + + if (xAxisUnwatch) { + xAxisUnwatch(); + xAxisUnwatch = null; + } + } + }); + } + } + }); }()); \ No newline at end of file diff --git a/rd_ui/app/views/visualizations/chart_editor.html b/rd_ui/app/views/visualizations/chart_editor.html index 71dee8ea0..dc8596768 100644 --- a/rd_ui/app/views/visualizations/chart_editor.html +++ b/rd_ui/app/views/visualizations/chart_editor.html @@ -7,5 +7,8 @@
        + + +
        \ No newline at end of file From 3cefa004cd35798d1195ce0259cefd5e81b8557d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 8 Apr 2014 11:20:52 +0300 Subject: [PATCH 475/540] Fix pie charts display --- rd_ui/app/scripts/ng_highchart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/ng_highchart.js b/rd_ui/app/scripts/ng_highchart.js index ee5e545c5..bdcd742d7 100644 --- a/rd_ui/app/scripts/ng_highchart.js +++ b/rd_ui/app/scripts/ng_highchart.js @@ -226,7 +226,7 @@ } } - if (chartOptions['xAxis']['type'] == 'category') { + if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') { if (!angular.isDefined(scope.series[0].data[0].name)) { // We need to make sure that for each category, each series has a value. var categories = _.union.apply(this, _.map(scope.series, function (s) { From a92ef02b075b2067c01d3d3908fb84d933a007d3 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 9 Apr 2014 15:24:19 +0300 Subject: [PATCH 476/540] Add select2 to the project --- rd_ui/app/index.html | 3 +++ rd_ui/app/scripts/app.js | 1 + rd_ui/bower.json | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index efc4000eb..67cac81dd 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -14,6 +14,7 @@ + @@ -106,6 +107,8 @@ + + diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 75594fb15..ac8a4b11d 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -8,6 +8,7 @@ angular.module('redash', [ 'redash.visualization', 'ui.codemirror', 'highchart', + 'ui.select2', 'angular-growl', 'angularMoment', 'ui.bootstrap', diff --git a/rd_ui/bower.json b/rd_ui/bower.json index d8bb06a0b..5f9c86d88 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -19,7 +19,8 @@ "pivottable": "https://github.com/arikfr/pivottable.git", "cornelius": "https://github.com/restorando/cornelius.git", "gridster": "0.2.0", - "mousetrap": "~1.4.6" + "mousetrap": "~1.4.6", + "angular-ui-select2": "~0.0.5" }, "devDependencies": { "angular-mocks": "~1.0.7", From 67ec5614e16aca470681a6c979b2f1114827aba2 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 9 Apr 2014 15:25:22 +0300 Subject: [PATCH 477/540] Add multi-filter option (#161) --- rd_ui/app/scripts/services/resources.js | 704 ++++++++++++------------ 1 file changed, 357 insertions(+), 347 deletions(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index d2ceeed0c..86521e4f2 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -1,357 +1,367 @@ (function () { - var QueryResult = function($resource, $timeout) { - var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}}); - var Job = $resource('/api/jobs/:id', {id: '@id'}); + var QueryResult = function ($resource, $timeout) { + var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}}); + var Job = $resource('/api/jobs/:id', {id: '@id'}); - var updateFunction = function (props) { - angular.extend(this, props); - if ('query_result' in props) { - this.status = "done"; - this.filters = undefined; - this.filterFreeze = undefined; + var updateFunction = function (props) { + angular.extend(this, props); + if ('query_result' in props) { + this.status = "done"; + this.filters = undefined; + this.filterFreeze = undefined; - _.each(this.query_result.data.rows, function (row) { - _.each(row, function (v, k) { - if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) { - row[k] = moment(v); - } - }); - }); - } else if (this.job.status == 3) { - this.status = "processing"; - } else { - this.status = undefined; + _.each(this.query_result.data.rows, function (row) { + _.each(row, function (v, k) { + if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) { + row[k] = moment(v); } - } - - function QueryResult(props) { - this.job = {}; - this.query_result = {}; - this.status = "waiting"; - this.filters = undefined; - this.filterFreeze = undefined; - - this.updatedAt = moment(); - - if (props) { - updateFunction.apply(this, [props]); - } - } - - var statuses = { - 1: "waiting", - 2: "processing", - 3: "done", - 4: "failed" - } - - QueryResult.prototype.update = updateFunction; - - QueryResult.prototype.getId = function() { - var id = null; - if ('query_result' in this) { - id = this.query_result.id; - } - return id; - } - - QueryResult.prototype.cancelExecution = function() { - Job.delete({id: this.job.id}); - } - - QueryResult.prototype.getStatus = function() { - return this.status || statuses[this.job.status]; - } - - QueryResult.prototype.getError = function() { - // TODO: move this logic to the server... - if (this.job.error == "None") { - return undefined; - } - - return this.job.error; - } - - QueryResult.prototype.getUpdatedAt = function() { - return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt; - } - - QueryResult.prototype.getRuntime = function() { - return this.query_result.runtime; - } - - QueryResult.prototype.getRawData = function() { - if (!this.query_result.data) { - return null; - } - - var data = this.query_result.data.rows; - - return data; - } - - QueryResult.prototype.getData = function() { - if (!this.query_result.data) { - return null; - } - - var filterValues = function(filters) { - if (!filters) { - return null; - } - - return _.reduce(filters, function(str, filter) { - return str + filter.current; - }, "") - } - - var filters = this.getFilters(); - var filterFreeze = filterValues(filters); - - if (this.filterFreeze != filterFreeze) { - this.filterFreeze = filterFreeze; - - if (filters) { - this.filteredData = _.filter(this.query_result.data.rows, function (row) { - return _.reduce(filters, function (memo, filter) { - return (memo && row[filter.name] == filter.current); - }, true); - }); - } else { - this.filteredData = this.query_result.data.rows; - } - } - - return this.filteredData; - } - - QueryResult.prototype.getChartData = function () { - var series = {}; - - _.each(this.getData(), function (row) { - var point = {}; - var seriesName = undefined; - var xValue = 0; - var yValues = {}; - - _.each(row, function (value, definition) { - var type = definition.split("::")[1]; - var name = definition.split("::")[0]; - - if (type == 'x') { - xValue = value; - point[type] = value; - } - - if (type == 'y') { - yValues[name] = value; - point[type] = value; - } - - if (type == 'series') { - seriesName = String(value); - } - }); - - var addPointToSeries = function(seriesName, point) { - if (series[seriesName] == undefined) { - series[seriesName] = { - name: seriesName, - type: 'column', - data: [] - } - } - - series[seriesName]['data'].push(point); - } - - if (seriesName === undefined) { - _.each(yValues, function(yValue, seriesName) { - addPointToSeries(seriesName, {'x': xValue, 'y': yValue}); - }); - } else { - addPointToSeries(seriesName, point); - } - }); - - _.each(series, function(series) { - series.data = _.sortBy(series.data, 'x'); - }); - - return _.values(series); - }; - - QueryResult.prototype.getColumns = function () { - if (this.columns == undefined && this.query_result.data) { - this.columns = _.map(this.query_result.data.columns, function(v) { - return v.name; - }); - } - - return this.columns; - } - - QueryResult.prototype.getColumnCleanName = function (column) { - var parts = column.split('::'); - var name = parts[1]; - if (parts[0] != '') { - // TODO: it's probably time to generalize this. - // see also getColumnFriendlyName - name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,''); - } - return name; - } - - QueryResult.prototype.getColumnFriendlyName = function (column) { - return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { - return a.toUpperCase(); - }); - } - - QueryResult.prototype.getColumnCleanNames = function () { - return _.map(this.getColumns(), function (col) { - return this.getColumnCleanName(col); - }, this); - } - - QueryResult.prototype.getColumnFriendlyNames = function () { - return _.map(this.getColumns(), function (col) { - return this.getColumnFriendlyName(col); - }, this); - } - - QueryResult.prototype.getFilters = function () { - if (!this.filters) { - this.prepareFilters(); - } - - return this.filters; - }; - - QueryResult.prototype.prepareFilters = function() { - var filterNames = []; - _.each(this.getColumns(), function (col) { - if (col.split('::')[1] == 'filter') { - filterNames.push(col); - } - }); - - var filterValues = []; - _.each(this.getRawData(), function (row) { - _.each(filterNames, function (filter, i) { - if (filterValues[i] == undefined) { - filterValues[i] = []; - } - filterValues[i].push(row[filter]); - }) - }); - - this.filters = _.map(filterNames, function (filter, i) { - var f = { - name: filter, - friendlyName: this.getColumnFriendlyName(filter), - values: _.uniq(filterValues[i]) - }; - - f.current = f.values[0]; - return f; - }, this); - } - - var refreshStatus = function(queryResult, query, ttl) { - Job.get({'id': queryResult.job.id}, function(response) { - queryResult.update(response); - - if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") { - QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) { - queryResult.update(response); - }); - } else if (queryResult.getStatus() != "failed") { - $timeout(function () { - refreshStatus(queryResult, query, ttl); - }, 3000); - } - }) - } - - QueryResult.getById = function (id) { - var queryResult = new QueryResult(); - - QueryResultResource.get({'id': id}, function (response) { - queryResult.update(response); - }); - - return queryResult; - } - - QueryResult.get = function (data_source_id, query, ttl) { - var queryResult = new QueryResult(); - - QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) { - queryResult.update(response); - - if ('job' in response) { - refreshStatus(queryResult, query, ttl); - } - }); - - return queryResult; - } - - return QueryResult; - }; - - var Query = function ($resource, QueryResult, DataSource) { - var Query = $resource('/api/queries/:id', {id: '@id'}); - - Query.newQuery = function() { - return new Query({ - query: "", - name: "New Query", - ttl: -1, - user: currentUser - }); - }; - - Query.prototype.getSourceLink = function() { - return '/queries/' + this.id + '/source'; - }; - - Query.prototype.getQueryResult = function(ttl) { - if (ttl == undefined) { - ttl = this.ttl; - } - - var queryResult = null; - if (this.latest_query_data && ttl != 0) { - queryResult = new QueryResult({'query_result': this.latest_query_data}); - } else if (this.latest_query_data_id && ttl != 0) { - queryResult = QueryResult.getById(this.latest_query_data_id); - } else if (this.data_source_id) { - queryResult = QueryResult.get(this.data_source_id, this.query, ttl); - } - - return queryResult; - }; - - return Query; - }; - - var DataSource = function($resource) { - var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}}); - - return DataSourceResource; + }); + }); + } else if (this.job.status == 3) { + this.status = "processing"; + } else { + this.status = undefined; + } } - var Widget = function($resource) { - var WidgetResource = $resource('/api/widgets/:id', {id: '@id'}); + function QueryResult(props) { + this.job = {}; + this.query_result = {}; + this.status = "waiting"; + this.filters = undefined; + this.filterFreeze = undefined; - return WidgetResource; + this.updatedAt = moment(); + + if (props) { + updateFunction.apply(this, [props]); + } } - angular.module('redash.services') - .factory('QueryResult', ['$resource', '$timeout', QueryResult]) - .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query]) - .factory('DataSource', ['$resource', DataSource]) - .factory('Widget', ['$resource', Widget]); + var statuses = { + 1: "waiting", + 2: "processing", + 3: "done", + 4: "failed" + } + + QueryResult.prototype.update = updateFunction; + + QueryResult.prototype.getId = function () { + var id = null; + if ('query_result' in this) { + id = this.query_result.id; + } + return id; + } + + QueryResult.prototype.cancelExecution = function () { + Job.delete({id: this.job.id}); + } + + QueryResult.prototype.getStatus = function () { + return this.status || statuses[this.job.status]; + } + + QueryResult.prototype.getError = function () { + // TODO: move this logic to the server... + if (this.job.error == "None") { + return undefined; + } + + return this.job.error; + } + + QueryResult.prototype.getUpdatedAt = function () { + return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt; + } + + QueryResult.prototype.getRuntime = function () { + return this.query_result.runtime; + } + + QueryResult.prototype.getRawData = function () { + if (!this.query_result.data) { + return null; + } + + var data = this.query_result.data.rows; + + return data; + } + + QueryResult.prototype.getData = function () { + if (!this.query_result.data) { + return null; + } + + var filterValues = function (filters) { + if (!filters) { + return null; + } + + return _.reduce(filters, function (str, filter) { + return str + filter.current; + }, "") + } + + var filters = this.getFilters(); + var filterFreeze = filterValues(filters); + + if (this.filterFreeze != filterFreeze) { + this.filterFreeze = filterFreeze; + + if (filters) { + this.filteredData = _.filter(this.query_result.data.rows, function (row) { + return _.reduce(filters, function (memo, filter) { + if (!_.isArray(filter.current)) { + filter.current = [filter.current]; + }; + return (memo && _.some(filter.current, function(v) { return v == row[filter.name] })); + }, true); + }); + } else { + this.filteredData = this.query_result.data.rows; + } + } + + return this.filteredData; + } + + QueryResult.prototype.getChartData = function () { + var series = {}; + + _.each(this.getData(), function (row) { + var point = {}; + var seriesName = undefined; + var xValue = 0; + var yValues = {}; + + _.each(row, function (value, definition) { + var type = definition.split("::")[1]; + var name = definition.split("::")[0]; + + if (type == 'x') { + xValue = value; + point[type] = value; + } + + if (type == 'y') { + yValues[name] = value; + point[type] = value; + } + + if (type == 'series') { + seriesName = String(value); + } + + if (type == 'multi-filter') { + seriesName = String(value); + } + }); + + var addPointToSeries = function (seriesName, point) { + if (series[seriesName] == undefined) { + series[seriesName] = { + name: seriesName, + type: 'column', + data: [] + } + } + + series[seriesName]['data'].push(point); + } + + if (seriesName === undefined) { + _.each(yValues, function (yValue, seriesName) { + addPointToSeries(seriesName, {'x': xValue, 'y': yValue}); + }); + } else { + addPointToSeries(seriesName, point); + } + }); + + _.each(series, function (series) { + series.data = _.sortBy(series.data, 'x'); + }); + + return _.values(series); + }; + + QueryResult.prototype.getColumns = function () { + if (this.columns == undefined && this.query_result.data) { + this.columns = _.map(this.query_result.data.columns, function (v) { + return v.name; + }); + } + + return this.columns; + } + + QueryResult.prototype.getColumnCleanName = function (column) { + var parts = column.split('::'); + var name = parts[1]; + if (parts[0] != '') { + // TODO: it's probably time to generalize this. + // see also getColumnFriendlyName + name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g, ''); + } + return name; + } + + QueryResult.prototype.getColumnFriendlyName = function (column) { + return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { + return a.toUpperCase(); + }); + } + + QueryResult.prototype.getColumnCleanNames = function () { + return _.map(this.getColumns(), function (col) { + return this.getColumnCleanName(col); + }, this); + } + + QueryResult.prototype.getColumnFriendlyNames = function () { + return _.map(this.getColumns(), function (col) { + return this.getColumnFriendlyName(col); + }, this); + } + + QueryResult.prototype.getFilters = function () { + if (!this.filters) { + this.prepareFilters(); + } + + return this.filters; + }; + + QueryResult.prototype.prepareFilters = function () { + var filters = []; + var filterTypes = ['filter', 'multi-filter']; + _.each(this.getColumns(), function (col) { + var type = col.split('::')[1] + if (_.contains(filterTypes, type)) { + // filter found + var filter = { + name: col, + friendlyName: this.getColumnFriendlyName(col), + values: [], + multiple: (type=='multi-filter') + } + filters.push(filter); + } + }, this); + + _.each(this.getRawData(), function (row) { + _.each(filters, function (filter) { + filter.values.push(row[filter.name]); + if (filter.values.length == 1) { + filter.current = row[filter.name]; + } + }) + }); + + _.each(filters, function(filter) { + filter.values = _.uniq(filter.values); + }); + + this.filters = filters; + } + + var refreshStatus = function (queryResult, query, ttl) { + Job.get({'id': queryResult.job.id}, function (response) { + queryResult.update(response); + + if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") { + QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) { + queryResult.update(response); + }); + } else if (queryResult.getStatus() != "failed") { + $timeout(function () { + refreshStatus(queryResult, query, ttl); + }, 3000); + } + }) + } + + QueryResult.getById = function (id) { + var queryResult = new QueryResult(); + + QueryResultResource.get({'id': id}, function (response) { + queryResult.update(response); + }); + + return queryResult; + } + + QueryResult.get = function (data_source_id, query, ttl) { + var queryResult = new QueryResult(); + + QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) { + queryResult.update(response); + + if ('job' in response) { + refreshStatus(queryResult, query, ttl); + } + }); + + return queryResult; + } + + return QueryResult; + }; + + var Query = function ($resource, QueryResult, DataSource) { + var Query = $resource('/api/queries/:id', {id: '@id'}); + + Query.newQuery = function () { + return new Query({ + query: "", + name: "New Query", + ttl: -1, + user: currentUser + }); + }; + + Query.prototype.getSourceLink = function () { + return '/queries/' + this.id + '/source'; + }; + + Query.prototype.getQueryResult = function (ttl) { + if (ttl == undefined) { + ttl = this.ttl; + } + + var queryResult = null; + if (this.latest_query_data && ttl != 0) { + queryResult = new QueryResult({'query_result': this.latest_query_data}); + } else if (this.latest_query_data_id && ttl != 0) { + queryResult = QueryResult.getById(this.latest_query_data_id); + } else if (this.data_source_id) { + queryResult = QueryResult.get(this.data_source_id, this.query, ttl); + } + + return queryResult; + }; + + return Query; + }; + + var DataSource = function ($resource) { + var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}}); + + return DataSourceResource; + } + + var Widget = function ($resource) { + var WidgetResource = $resource('/api/widgets/:id', {id: '@id'}); + + return WidgetResource; + } + + angular.module('redash.services') + .factory('QueryResult', ['$resource', '$timeout', QueryResult]) + .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query]) + .factory('DataSource', ['$resource', DataSource]) + .factory('Widget', ['$resource', Widget]); })(); From eca62cd1f266cb17de1bd6f04698bfaab5cb3378 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 9 Apr 2014 15:25:49 +0300 Subject: [PATCH 478/540] Use select2 for filters, for autocomplete and multiple selection (#161, #160) --- rd_ui/app/views/visualizations/filters.html | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/rd_ui/app/views/visualizations/filters.html b/rd_ui/app/views/visualizations/filters.html index 3d718a944..52bd92bd7 100644 --- a/rd_ui/app/views/visualizations/filters.html +++ b/rd_ui/app/views/visualizations/filters.html @@ -1,12 +1,7 @@
        -
        - - +
        +
        \ No newline at end of file From 01b908539be181a85e982edefa843af2cb8a047b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 9 Apr 2014 16:27:16 +0300 Subject: [PATCH 479/540] Show filter name --- rd_ui/app/views/visualizations/filters.html | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/views/visualizations/filters.html b/rd_ui/app/views/visualizations/filters.html index 52bd92bd7..52fd7079a 100644 --- a/rd_ui/app/views/visualizations/filters.html +++ b/rd_ui/app/views/visualizations/filters.html @@ -1,5 +1,6 @@
        + {{filter.friendlyName}}: From 718577f56555b5e64dc2f5d243ee1da97a772c80 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 12:29:07 +0300 Subject: [PATCH 480/540] Events reporting from client side. --- .../scripts/controllers/admin_controllers.js | 35 +- rd_ui/app/scripts/controllers/controllers.js | 6 +- rd_ui/app/scripts/controllers/dashboard.js | 16 +- rd_ui/app/scripts/controllers/query_source.js | 9 +- rd_ui/app/scripts/controllers/query_view.js | 85 ++--- .../directives/dashboard_directives.js | 12 +- rd_ui/app/scripts/services/notifications.js | 5 +- rd_ui/app/scripts/services/services.js | 30 +- rd_ui/app/scripts/visualizations/base.js | 324 +++++++++--------- 9 files changed, 285 insertions(+), 237 deletions(-) diff --git a/rd_ui/app/scripts/controllers/admin_controllers.js b/rd_ui/app/scripts/controllers/admin_controllers.js index f05c67caa..170c5ed38 100644 --- a/rd_ui/app/scripts/controllers/admin_controllers.js +++ b/rd_ui/app/scripts/controllers/admin_controllers.js @@ -1,23 +1,24 @@ (function () { - var AdminStatusCtrl = function ($scope, $http, $timeout) { - $scope.$parent.pageTitle = "System Status"; + var AdminStatusCtrl = function ($scope, Events, $http, $timeout) { + Events.record(currentUser, "view", "page", "admin/status"); + $scope.$parent.pageTitle = "System Status"; - var refresh = function () { - $scope.refresh_time = moment().add('minutes', 1); - $http.get('/status.json').success(function (data) { - $scope.workers = data.workers; - delete data.workers; - $scope.manager = data.manager; - delete data.manager; - $scope.status = data; - }); + var refresh = function () { + $scope.refresh_time = moment().add('minutes', 1); + $http.get('/status.json').success(function (data) { + $scope.workers = data.workers; + delete data.workers; + $scope.manager = data.manager; + delete data.manager; + $scope.status = data; + }); - $timeout(refresh, 59 * 1000); - }; + $timeout(refresh, 59 * 1000); + }; - refresh(); - } + refresh(); + } - angular.module('redash.admin_controllers', []) - .controller('AdminStatusCtrl', ['$scope', '$http', '$timeout', AdminStatusCtrl]) + angular.module('redash.admin_controllers', []) + .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl]) })(); diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index d68dfe92c..dcb4672b7 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -136,11 +136,13 @@ }); } - var IndexCtrl = function($scope, Dashboard) { + var IndexCtrl = function($scope, Events, Dashboard) { + Events.record(currentUser, "view", "page", "homepage"); $scope.$parent.pageTitle = "Home"; $scope.archiveDashboard = function(dashboard) { if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) { + Events.record(currentUser, "archive", "dashboard", dashboard.id); dashboard.$delete(function() { $scope.$parent.reloadDashboards(); }); @@ -150,6 +152,6 @@ angular.module('redash.controllers', []) .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl]) + .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl]) .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index e427ed954..34b46a3c7 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -1,5 +1,7 @@ (function() { - var DashboardCtrl = function($scope, $routeParams, $http, $timeout, Dashboard) { + var DashboardCtrl = function($scope, Events, $routeParams, $http, $timeout, Dashboard) { + Events.record(currentUser, "view", "dashboard", dashboard.id); + $scope.refreshEnabled = false; $scope.refreshRate = 60; $scope.dashboard = Dashboard.get({ @@ -35,6 +37,8 @@ $scope.triggerRefresh = function() { $scope.refreshEnabled = !$scope.refreshEnabled; + Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled}); + if ($scope.refreshEnabled) { var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) { return widget.visualization.query.ttl; @@ -47,12 +51,14 @@ }; }; - var WidgetCtrl = function($scope, $http, $location, Query) { + var WidgetCtrl = function($scope, Events, $http, $location, Query) { $scope.deleteWidget = function() { if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) { return; } + Events.record(currentUser, "delete", "widget", $scope.widget.id); + $http.delete('/api/widgets/' + $scope.widget.id).success(function() { $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) { return _.filter(row, function(widget) { @@ -62,6 +68,8 @@ }); }; + // TODO: fire event for query view for each query + $scope.query = new Query($scope.widget.visualization.query); $scope.queryResult = $scope.query.getQueryResult(); @@ -72,7 +80,7 @@ }; angular.module('redash.controllers') - .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) - .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl]) + .controller('DashboardCtrl', ['$scope', 'Events', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) + .controller('WidgetCtrl', ['$scope', 'Events', '$http', '$location', 'Query', WidgetCtrl]) })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/query_source.js b/rd_ui/app/scripts/controllers/query_source.js index f973ee84b..075b9dee9 100644 --- a/rd_ui/app/scripts/controllers/query_source.js +++ b/rd_ui/app/scripts/controllers/query_source.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - function QuerySourceCtrl($controller, $scope, $location, Query, Visualization, KeyboardShortcuts) { + function QuerySourceCtrl(Events, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) { // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); // TODO: @@ -9,6 +9,8 @@ // Obviously it shouldn't be repeated, but we got bigger fish to fry. var DEFAULT_TAB = 'table'; + Events.record(currentUser, 'view_source', 'query', $scope.query.id); + var isNewQuery = !$scope.query.id, queryText = $scope.query.query, // ref to QueryViewCtrl.saveQuery @@ -47,6 +49,7 @@ }; $scope.duplicateQuery = function() { + Events.record(currentUser, 'fork', 'query', $scope.query.id); $scope.query.id = null; $scope.query.ttl = -1; @@ -62,6 +65,8 @@ $scope.deleteVisualization = function($e, vis) { $e.preventDefault(); if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { + Events.record(currentUser, 'delete', 'visualization', vis.id); + Visualization.delete(vis); if ($scope.selectedTab == vis.id) { $scope.selectedTab = DEFAULT_TAB; @@ -94,7 +99,7 @@ } angular.module('redash.controllers').controller('QuerySourceCtrl', [ - '$controller', '$scope', '$location', 'Query', + 'Events', '$controller', '$scope', '$location', 'Query', 'Visualization', 'KeyboardShortcuts', QuerySourceCtrl ]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 464e42fb6..46e0fd902 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -1,9 +1,11 @@ (function() { 'use strict'; - function QueryViewCtrl($scope, $route, $location, notifications, growl, Query, DataSource) { + function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) { var DEFAULT_TAB = 'table'; + Events.record(currentUser, 'view', 'query', $scope.query.id); + $scope.query = $route.current.locals.query; $scope.queryResult = $scope.query.getQueryResult(); $scope.queryExecuting = false; @@ -37,15 +39,16 @@ growl.addSuccessMessage(options.successMessage); }, function(httpResponse) { growl.addErrorMessage(options.errorMessage); - }) - .$promise; + }).$promise; } $scope.saveDescription = function() { + Events.record(currentUser, 'edit_description', 'query', $scope.query.id); $scope.saveQuery(undefined, {'description': $scope.query.description}); }; $scope.saveName = function() { + Events.record(currentUser, 'edit_name', 'query', $scope.query.id); $scope.saveQuery(undefined, {'name': $scope.query.name}); }; @@ -53,23 +56,26 @@ $scope.queryResult = $scope.query.getQueryResult(0); $scope.lockButton(true); $scope.cancelling = false; + Events.record(currentUser, 'execute', 'query', $scope.query.id); }; $scope.cancelExecution = function() { $scope.cancelling = true; $scope.queryResult.cancelExecution(); + Events.record(currentUser, 'cancel_execute', 'query', $scope.query.id); }; $scope.updateDataSource = function() { - $scope.query.latest_query_data = null; - $scope.query.latest_query_data_id = null; - Query.save({ - 'id': $scope.query.id, - 'data_source_id': $scope.query.data_source_id, - 'latest_query_data_id': null - }); + Events.record(currentUser, 'update_data_source', 'query', $scope.query.id); + $scope.query.latest_query_data = null; + $scope.query.latest_query_data_id = null; + Query.save({ + 'id': $scope.query.id, + 'data_source_id': $scope.query.data_source_id, + 'latest_query_data_id': null + }); - $scope.executeQuery(); + $scope.executeQuery(); }; $scope.setVisualizationTab = function (visualization) { @@ -81,38 +87,36 @@ $scope.$parent.pageTitle = $scope.query.name; }); - $scope.$watch('queryResult && queryResult.getError()', - function(newError, oldError) { - if (newError == undefined) { - return; - } + $scope.$watch('queryResult && queryResult.getError()', function(newError, oldError) { + if (newError == undefined) { + return; + } - if (oldError == undefined && newError != undefined) { - $scope.lockButton(false); - } - }); + if (oldError == undefined && newError != undefined) { + $scope.lockButton(false); + } + }); - $scope.$watch('queryResult && queryResult.getData()', - function(data, oldData) { - if (!data) { - return; - } + $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) { + if (!data) { + return; + } - $scope.filters = $scope.queryResult.getFilters(); + $scope.filters = $scope.queryResult.getFilters(); - if ($scope.queryResult.getId() == null) { - $scope.dataUri = ""; - } else { - $scope.dataUri = - '/api/queries/' + $scope.query.id + '/results/' + - $scope.queryResult.getId() + '.csv'; + if ($scope.queryResult.getId() == null) { + $scope.dataUri = ""; + } else { + $scope.dataUri = + '/api/queries/' + $scope.query.id + '/results/' + + $scope.queryResult.getId() + '.csv'; - $scope.dataFilename = - $scope.query.name.replace(" ", "_") + - moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + - ".csv"; - } - }); + $scope.dataFilename = + $scope.query.name.replace(" ", "_") + + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + + ".csv"; + } + }); $scope.$watch("queryResult && queryResult.getStatus()", function(status) { if (!status) { @@ -139,11 +143,14 @@ $scope.$watch(function() { return $location.hash() }, function(hash) { + if (hash == 'pivot') { + Events.record(currentUser, 'pivot', 'query', $scope.query && $scope.query.id); + } $scope.selectedTab = hash || DEFAULT_TAB; }); }; angular.module('redash.controllers') .controller('QueryViewCtrl', - ['$scope', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]); + ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]); })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js index d39ae2c37..c1bc2dde3 100644 --- a/rd_ui/app/scripts/directives/dashboard_directives.js +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -3,8 +3,8 @@ var directives = angular.module('redash.directives'); - directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', - function($http, $location, $timeout, Dashboard) { + directives.directive('editDashboardForm', ['Events', '$http', '$location', '$timeout', 'Dashboard', + function(Events, $http, $location, $timeout, Dashboard) { return { restrict: 'E', scope: { @@ -54,7 +54,6 @@ _.each(layout, function(item) { var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name); gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row); - }); } }); @@ -89,14 +88,17 @@ $scope.dashboard = new Dashboard(response); $scope.saveInProgress = false; $(element).modal('hide'); - }) + }); + Events.record(currentUser, 'edit', 'dashboard', $scope.dashboard.id); } else { + $http.post('/api/dashboards', { 'name': $scope.dashboard.name }).success(function(response) { $(element).modal('hide'); $location.path('/dashboard/' + response.slug).replace(); - }) + }); + Events.record(currentUser, 'create', 'dashboard'); } } diff --git a/rd_ui/app/scripts/services/notifications.js b/rd_ui/app/scripts/services/notifications.js index 455569955..dfb5ae422 100644 --- a/rd_ui/app/scripts/services/notifications.js +++ b/rd_ui/app/scripts/services/notifications.js @@ -1,5 +1,5 @@ (function () { - var notifications = function () { + var notifications = function (Events) { var notificationService = {}; var lastNotification = null; @@ -40,6 +40,7 @@ notification.onclick = function () { window.focus(); this.cancel(); + Events.record(currentUser, 'click', 'notification'); }; notification.show() @@ -49,5 +50,5 @@ } angular.module('redash.services') - .factory('notifications', notifications); + .factory('notifications', ['Events', notifications]); })(); diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index feb2189df..a8dd79bb1 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -1,24 +1,40 @@ -(function() { +(function () { 'use strict' function KeyboardShortcuts() { this.bind = function bind(keymap) { - _.forEach(keymap, function(fn, key) { - Mousetrap.bindGlobal(key, function(e) { - e.preventDefault(); - fn(); + _.forEach(keymap, function (fn, key) { + Mousetrap.bindGlobal(key, function (e) { + e.preventDefault(); + fn(); }); }); } this.unbind = function unbind(keymap) { - _.forEach(keymap, function(fn, key) { + _.forEach(keymap, function (fn, key) { Mousetrap.unbind(key); }); } } + function Events($http) { + this.record = function (user, action, object_type, object_id, additional_properties) { + + var event = { + "user_id": user.id, + "action": action, + "object_type": object_type, + "object_id": object_id + }; + _.extend(event, additional_properties); + + $http.post('/api/events', [event]); + }; + } + angular.module('redash.services', []) - .service('KeyboardShortcuts', [KeyboardShortcuts]) + .service('KeyboardShortcuts', [KeyboardShortcuts]) + .service('Events', ['$http', Events]) })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index 2bc770b2f..f733b1f6e 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -1,170 +1,176 @@ (function () { - var VisualizationProvider = function() { - this.visualizations = {}; - this.visualizationTypes = {}; - var defaultConfig = { - defaultOptions: {}, - skipTypes: false, - editorTemplate: null - } - - this.registerVisualization = function(config) { - var visualization = _.extend({}, defaultConfig, config); - - // TODO: this is prone to errors; better refactor. - if (_.isEmpty(this.visualizations)) { - this.defaultVisualization = visualization; - } - - this.visualizations[config.type] = visualization; - - if (!config.skipTypes) { - this.visualizationTypes[config.name] = config.type; - }; - }; - - this.getSwitchTemplate = function(property) { - var pattern = /(<[a-zA-Z0-9-]*?)( |>)/ - - var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) { - if (visualization[property]) { - var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2'; - var template = visualization[property].replace(pattern, ngSwitch); - - return templates + "\n" + template; - } - - return templates; - }, ""); - - mergedTemplates = '
        '+ mergedTemplates + "
        "; - - return mergedTemplates; - } - - this.$get = ['$resource', function($resource) { - var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); - Visualization.visualizations = this.visualizations; - Visualization.visualizationTypes = this.visualizationTypes; - Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate'); - Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); - Visualization.defaultVisualization = this.defaultVisualization; - - return Visualization; - }]; - }; - - var VisualizationRenderer = function(Visualization) { - return { - restrict: 'E', - scope: { - visualization: '=', - queryResult: '=' - }, - // TODO: using switch here (and in the options editor) might introduce errors and bad - // performance wise. It's better to eventually show the correct template based on the - // visualization type and not make the browser render all of them. - template: '\n' + Visualization.renderVisualizationsTemplate, - replace: false, - link: function(scope) { - scope.$watch('queryResult && queryResult.getFilters()', function(filters) { - if (filters) { - scope.filters = filters; - } - }); - } - } - }; - - var VisualizationOptionsEditor = function(Visualization) { - return { - restrict: 'E', - template: Visualization.editorTemplate, - replace: false - } - }; - - var Filters = function() { - return { - restrict: 'E', - templateUrl: '/views/visualizations/filters.html' - } + var VisualizationProvider = function () { + this.visualizations = {}; + this.visualizationTypes = {}; + var defaultConfig = { + defaultOptions: {}, + skipTypes: false, + editorTemplate: null } - var EditVisualizationForm = function(Visualization, growl) { - return { - restrict: 'E', - templateUrl: '/views/visualizations/edit_visualization.html', - replace: true, - scope: { - query: '=', - queryResult: '=', - visualization: '=?', - onNewSuccess: '=?' - }, - link: function (scope, element, attrs) { - scope.editRawOptions = currentUser.hasPermission('edit_raw_chart'); - scope.visTypes = Visualization.visualizationTypes; + this.registerVisualization = function (config) { + var visualization = _.extend({}, defaultConfig, config); - scope.newVisualization = function(q) { - return { - 'query_id': q.id, - 'type': Visualization.defaultVisualization.type, - 'name': Visualization.defaultVisualization.name, - 'description': q.description || '', - 'options': Visualization.defaultVisualization.defaultOptions - }; - } + // TODO: this is prone to errors; better refactor. + if (_.isEmpty(this.visualizations)) { + this.defaultVisualization = visualization; + } - if (!scope.visualization) { - // create new visualization - // wait for query to load to populate with defaults - var unwatch = scope.$watch('query', function (q) { - if (q && q.id) { - unwatch(); + this.visualizations[config.type] = visualization; - scope.visualization = scope.newVisualization(q); - } - }, true); - } - - scope.$watch('visualization.type', function (type, oldType) { - // if not edited by user, set name to match type - if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) { - // poor man's titlecase - scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase(); - } - }); - - scope.submit = function () { - Visualization.save(scope.visualization, function success(result) { - growl.addSuccessMessage("Visualization saved"); - - scope.visualization = scope.newVisualization(scope.query); - - var visIds = _.pluck(scope.query.visualizations, 'id'); - var index = visIds.indexOf(result.id); - if (index > -1) { - scope.query.visualizations[index] = result; - } else { - // new visualization - scope.query.visualizations.push(result); - scope.onNewSuccess && scope.onNewSuccess(result); - } - }, function error() { - growl.addErrorMessage("Visualization could not be saved"); - }); - }; - } - } + if (!config.skipTypes) { + this.visualizationTypes[config.name] = config.type; + } + ; }; + this.getSwitchTemplate = function (property) { + var pattern = /(<[a-zA-Z0-9-]*?)( |>)/ + + var mergedTemplates = _.reduce(this.visualizations, function (templates, visualization) { + if (visualization[property]) { + var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2'; + var template = visualization[property].replace(pattern, ngSwitch); + + return templates + "\n" + template; + } + + return templates; + }, ""); + + mergedTemplates = '
        ' + mergedTemplates + "
        "; + + return mergedTemplates; + } + + this.$get = ['$resource', function ($resource) { + var Visualization = $resource('/api/visualizations/:id', {id: '@id'}); + Visualization.visualizations = this.visualizations; + Visualization.visualizationTypes = this.visualizationTypes; + Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate'); + Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); + Visualization.defaultVisualization = this.defaultVisualization; + + return Visualization; + }]; + }; + + var VisualizationRenderer = function (Visualization) { + return { + restrict: 'E', + scope: { + visualization: '=', + queryResult: '=' + }, + // TODO: using switch here (and in the options editor) might introduce errors and bad + // performance wise. It's better to eventually show the correct template based on the + // visualization type and not make the browser render all of them. + template: '\n' + Visualization.renderVisualizationsTemplate, + replace: false, + link: function (scope) { + scope.$watch('queryResult && queryResult.getFilters()', function (filters) { + if (filters) { + scope.filters = filters; + } + }); + } + } + }; + + var VisualizationOptionsEditor = function (Visualization) { + return { + restrict: 'E', + template: Visualization.editorTemplate, + replace: false + } + }; + + var Filters = function () { + return { + restrict: 'E', + templateUrl: '/views/visualizations/filters.html' + } + } + + var EditVisualizationForm = function (Events, Visualization, growl) { + return { + restrict: 'E', + templateUrl: '/views/visualizations/edit_visualization.html', + replace: true, + scope: { + query: '=', + queryResult: '=', + visualization: '=?', + onNewSuccess: '=?' + }, + link: function (scope, element, attrs) { + scope.editRawOptions = currentUser.hasPermission('edit_raw_chart'); + scope.visTypes = Visualization.visualizationTypes; + + scope.newVisualization = function (q) { + return { + 'query_id': q.id, + 'type': Visualization.defaultVisualization.type, + 'name': Visualization.defaultVisualization.name, + 'description': q.description || '', + 'options': Visualization.defaultVisualization.defaultOptions + }; + } + + if (!scope.visualization) { + // create new visualization + // wait for query to load to populate with defaults + var unwatch = scope.$watch('query', function (q) { + if (q && q.id) { + unwatch(); + + scope.visualization = scope.newVisualization(q); + } + }, true); + } + + scope.$watch('visualization.type', function (type, oldType) { + // if not edited by user, set name to match type + if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) { + // poor man's titlecase + scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase(); + } + }); + + scope.submit = function () { + if (scope.visualization.id) { + Events.record(currentUser, "update", "visualization", scope.visualization.id, {'type': scope.visualization.type}); + } else { + Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type}); + } + + Visualization.save(scope.visualization, function success(result) { + growl.addSuccessMessage("Visualization saved"); + + scope.visualization = scope.newVisualization(scope.query); + + var visIds = _.pluck(scope.query.visualizations, 'id'); + var index = visIds.indexOf(result.id); + if (index > -1) { + scope.query.visualizations[index] = result; + } else { + // new visualization + scope.query.visualizations.push(result); + scope.onNewSuccess && scope.onNewSuccess(result); + } + }, function error() { + growl.addErrorMessage("Visualization could not be saved"); + }); + }; + } + } + }; - angular.module('redash.visualization', []) - .provider('Visualization', VisualizationProvider) - .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) - .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor]) - .directive('filters', Filters) - .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm]) + angular.module('redash.visualization', []) + .provider('Visualization', VisualizationProvider) + .directive('visualizationRenderer', ['Visualization', VisualizationRenderer]) + .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor]) + .directive('filters', Filters) + .directive('editVisulatizationForm', ['Events', 'Visualization', 'growl', EditVisualizationForm]) })(); From c84f18449baba577677f679d0eb4d8adb30bf600 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 12:29:21 +0300 Subject: [PATCH 481/540] Events end point. --- redash/controllers.py | 11 +++++++++++ redash/events.py | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100644 redash/events.py diff --git a/redash/controllers.py b/redash/controllers.py index 7ebde0a84..ceb140412 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -17,6 +17,7 @@ from flask.ext.restful import Resource, abort from flask_login import current_user, login_user, logout_user import sqlparse +import events from permissions import require_permission from redash import settings, utils from redash import data @@ -129,6 +130,16 @@ class BaseResource(Resource): return current_user._get_current_object() +class EventAPI(BaseResource): + def post(self): + events_list = request.get_json(force=True) + for event in events_list: + events.record_event(event) + + +api.add_resource(EventAPI, '/api/events', endpoint='events') + + class DataSourceListAPI(BaseResource): def get(self): data_sources = [ds.to_dict() for ds in models.DataSource.select()] diff --git a/redash/events.py b/redash/events.py new file mode 100644 index 000000000..28ca72b87 --- /dev/null +++ b/redash/events.py @@ -0,0 +1,8 @@ +import logging +import json + +logger = logging.getLogger("events") + + +def record_event(event): + logger.info(json.dumps(event)) \ No newline at end of file From 65e8bef22c9600bde237868dd70e748aa1875800 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 12:55:02 +0300 Subject: [PATCH 482/540] Improved logging output. --- redash/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 9078b39b3..30104d638 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -11,8 +11,15 @@ from redash import settings, utils __version__ = '0.3.5' -logging.getLogger().addHandler(logging.StreamHandler()) -logging.getLogger().setLevel(settings.LOG_LEVEL) + +def setup_logging(): + handler = logging.StreamHandler() + formatter = logging.Formatter('[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s') + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(settings.LOG_LEVEL) + +setup_logging() app = Flask(__name__, template_folder=settings.STATIC_ASSETS_PATH, From bd67c2ff219edfe9c443b3d36adf14329f7454e0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 12:55:31 +0300 Subject: [PATCH 483/540] Bump version (about time...) --- redash/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/__init__.py b/redash/__init__.py index 30104d638..061bcc298 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -9,7 +9,7 @@ import redis from statsd import StatsClient from redash import settings, utils -__version__ = '0.3.5' +__version__ = '0.3.6' def setup_logging(): From 8615429e0c6615ee8ae5968bec45dc94f2064b96 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 13:02:24 +0300 Subject: [PATCH 484/540] Logging setup for events. --- redash/events.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/redash/events.py b/redash/events.py index 28ca72b87..c7c996d95 100644 --- a/redash/events.py +++ b/redash/events.py @@ -1,7 +1,21 @@ import logging import json -logger = logging.getLogger("events") +logger = logging.getLogger("redash.events") +logger.propagate = False + + +def setup_logging(log_path, console_output=False): + fh = logging.FileHandler(log_path) + formatter = logging.Formatter('%(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + + if console_output: + handler = logging.StreamHandler() + formatter = logging.Formatter('[%(name)s] %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) def record_event(event): From 6d62f0d2c9242ac3dba3da72f9265e79164782d1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 13:02:40 +0300 Subject: [PATCH 485/540] Setup events logging from settings. --- redash/__init__.py | 4 ++++ redash/settings.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/redash/__init__.py b/redash/__init__.py index 061bcc298..5ef034048 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -7,6 +7,7 @@ from flask_peewee.db import Database import redis from statsd import StatsClient +import events from redash import settings, utils __version__ = '0.3.6' @@ -19,6 +20,9 @@ def setup_logging(): logging.getLogger().addHandler(handler) logging.getLogger().setLevel(settings.LOG_LEVEL) + if settings.EVENTS_LOG_PATH: + events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT) + setup_logging() app = Flask(__name__, diff --git a/redash/settings.py b/redash/settings.py index 6f437632b..a225c1206 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -65,4 +65,6 @@ STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") +EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "") +EVENTS_CONSOLE_OUTPUT = parse_boolean(os.environ.get("REDASH_EVENTS_CONSOLE_OUTPUT", "false")) ANALYTICS = os.environ.get("REDASH_ANALYTICS", "") \ No newline at end of file From 2c7a6004c03361bffb505255cb9f90ca018a7bf1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 13:02:52 +0300 Subject: [PATCH 486/540] Pass timestamp with event. --- rd_ui/app/scripts/services/services.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index a8dd79bb1..975adc4c4 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -26,7 +26,8 @@ "user_id": user.id, "action": action, "object_type": object_type, - "object_id": object_id + "object_id": object_id, + "timestamp": Date.now()/1000.0 }; _.extend(event, additional_properties); From d7e1328fc02e49a4e925fbae5cb8e3baf3a19e42 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 13:04:03 +0300 Subject: [PATCH 487/540] Make it possible to log events to stdout without logging to file. --- redash/__init__.py | 3 +-- redash/events.py | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 5ef034048..ecc6a777c 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -20,8 +20,7 @@ def setup_logging(): logging.getLogger().addHandler(handler) logging.getLogger().setLevel(settings.LOG_LEVEL) - if settings.EVENTS_LOG_PATH: - events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT) + events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT) setup_logging() diff --git a/redash/events.py b/redash/events.py index c7c996d95..feef260bc 100644 --- a/redash/events.py +++ b/redash/events.py @@ -6,10 +6,11 @@ logger.propagate = False def setup_logging(log_path, console_output=False): - fh = logging.FileHandler(log_path) - formatter = logging.Formatter('%(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) + if log_path: + fh = logging.FileHandler(log_path) + formatter = logging.Formatter('%(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) if console_output: handler = logging.StreamHandler() From 881e44fbb692b283ec3d4bbf596ce06f4bd12fc6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 10 Apr 2014 15:10:39 +0300 Subject: [PATCH 488/540] Fix: access query after it's assigned --- rd_ui/app/scripts/controllers/query_view.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 46e0fd902..b957905ee 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -4,9 +4,8 @@ function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) { var DEFAULT_TAB = 'table'; - Events.record(currentUser, 'view', 'query', $scope.query.id); - $scope.query = $route.current.locals.query; + Events.record(currentUser, 'view', 'query', $scope.query.id); $scope.queryResult = $scope.query.getQueryResult(); $scope.queryExecuting = false; From 433e004295974877654d8fef6eb020fc59d5b34f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 12 Apr 2014 16:36:20 +0300 Subject: [PATCH 489/540] Set TTL on finishsed jobs (fix #106) --- redash/data/worker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/redash/data/worker.py b/redash/data/worker.py index 113325929..6ab49cf11 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -173,6 +173,9 @@ class Job(RedisObject): super(Job, self).save(pipe) + def expire(self, expire_time): + self.redis_connection.expire(self._redis_key(self.id), expire_time) + def processing(self, process_id): self.update(status=self.PROCESSING, process_id=process_id, @@ -279,6 +282,8 @@ class Worker(threading.Thread): self.name, job_id) job.done(None, "Interrupted/Cancelled while running.") + job.expire(24 * 3600) + logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) From d9af5d3943bd6515923ec6131197c776fd0b1b97 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 12 Apr 2014 16:43:33 +0300 Subject: [PATCH 490/540] Fix: user should be able to cancel query even if process not existing already (#8). --- redash/data/worker.py | 10 +++++++--- tests/test_job.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/redash/data/worker.py b/redash/data/worker.py index 6ab49cf11..2df32a665 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -157,9 +157,13 @@ class Job(RedisObject): return if self.status == self.PROCESSING: - os.kill(self.process_id, signal.SIGINT) - else: - self.done(None, "Interrupted/Cancelled while running.") + try: + os.kill(self.process_id, signal.SIGINT) + except OSError as e: + logging.warning("[%s] Tried to cancel job but os.kill failed (pid=%d, error=%s)", + self.id, self.process_id, e) + + self.done(None, "Interrupted/Cancelled while running.") def save(self, pipe=None): if not pipe: diff --git a/tests/test_job.py b/tests/test_job.py index 68efa013e..f178f978b 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -98,3 +98,14 @@ class TestJob(TestCase): loaded_job = Job.load(redis_connection, job.id) self.assertEquals(loaded_job.query, unicode_query) + def test_cancel_job_with_no_process(self): + job = Job(redis_connection, query=self.query, priority=self.priority) + job.status = Job.PROCESSING + job.process_id = 699999 + job.save() + + job.cancel() + + job = Job.load(redis_connection, job.id) + + self.assertEquals(job.status, Job.FAILED) From 9f9d78fd7a5011f24b5e12249f1276010293d877 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 13 Apr 2014 14:14:55 +0300 Subject: [PATCH 491/540] Update upload script to include checksums --- bin/upload_version.py | 54 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/bin/upload_version.py b/bin/upload_version.py index 3a074676d..cb02e4d99 100644 --- a/bin/upload_version.py +++ b/bin/upload_version.py @@ -3,30 +3,44 @@ import os import sys import json import requests +import subprocess + + +def capture_output(command): + proc = subprocess.Popen(command, stdout=subprocess.PIPE) + return proc.stdout.read() + if __name__ == '__main__': - version = sys.argv[1] - filepath = sys.argv[2] - filename = filepath.split('/')[-1] - github_token = os.environ['GITHUB_TOKEN'] - auth = (github_token, 'x-oauth-basic') - commit_sha = os.environ['CIRCLE_SHA1'] + version = sys.argv[1] + filepath = sys.argv[2] + filename = filepath.split('/')[-1] + github_token = os.environ['GITHUB_TOKEN'] + auth = (github_token, 'x-oauth-basic') + commit_sha = os.environ['CIRCLE_SHA1'] - params = json.dumps({ - 'tag_name': 'v{0}'.format(version), - 'name': 're:dash v{0}'.format(version), - 'target_commitish': commit_sha, - 'prerelease': True - }) + commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha]) + file_md5_checksum = capture_output(["md5sum", filename]).split()[0] + file_sha256_checksum = capture_output(["sha256sum", filename]).split()[0] + version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum) - response = requests.post('https://api.github.com/repos/everythingme/redash/releases', - data=params, - auth=auth) + params = json.dumps({ + 'tag_name': 'v{0}'.format(version), + 'name': 're:dash v{0}'.format(version), + 'body': version_body, + 'target_commitish': commit_sha, + 'prerelease': True + }) - upload_url = response.json()['upload_url'] - upload_url = upload_url.replace('{?name}', '') + response = requests.post('https://api.github.com/repos/everythingme/redash/releases', + data=params, + auth=auth) - with open(filepath) as file_content: - headers = {'Content-Type': 'application/gzip'} - response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False) + upload_url = response.json()['upload_url'] + upload_url = upload_url.replace('{?name}', '') + + with open(filepath) as file_content: + headers = {'Content-Type': 'application/gzip'} + response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, + headers=headers, verify=False) From 8178900d56abe40124f7685858e640694c87f3af Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 13 Apr 2014 16:11:21 +0300 Subject: [PATCH 492/540] Copy select2 assets, as the grunt pipeline skips them --- rd_ui/Gruntfile.js | 1 + rd_ui/app/styles/select2-spinner.gif | Bin 0 -> 1849 bytes rd_ui/app/styles/select2.png | Bin 0 -> 613 bytes rd_ui/app/styles/select2x2.png | Bin 0 -> 845 bytes 4 files changed, 1 insertion(+) create mode 100644 rd_ui/app/styles/select2-spinner.gif create mode 100644 rd_ui/app/styles/select2.png create mode 100644 rd_ui/app/styles/select2x2.png diff --git a/rd_ui/Gruntfile.js b/rd_ui/Gruntfile.js index 5d9c59df8..9d41d54e6 100644 --- a/rd_ui/Gruntfile.js +++ b/rd_ui/Gruntfile.js @@ -249,6 +249,7 @@ module.exports = function (grunt) { '.htaccess', 'bower_components/**/*', 'images/{,*/}*.{gif,webp}', + 'styles/{,*/}*.{png,gif}', 'fonts/*' ] }, { diff --git a/rd_ui/app/styles/select2-spinner.gif b/rd_ui/app/styles/select2-spinner.gif new file mode 100644 index 0000000000000000000000000000000000000000..5b33f7e54f4e55b6b8774d86d96895db9af044b4 GIT binary patch literal 1849 zcma*odr(tX9tZI2z31lM+(&YVk%mZ}5P~KlG2s=WSbGzm0!x7^P##Mnh7t-jP!X0Q zk_SQ}Po-L1tlDK;6l?(>v)e5ZBQx4|Y-Q?nr@Px3?9h(3ZWr3^tj=`TP57gKr87N$ zp2wWee1GRRCwo_xahnw)5cxNPJbCg2L6DV|6`#+yw6v6!mDS$f9-JvFD^n;GQ&UrZ zzh5jCkByB101O60U0q#p_1BM>Cv-vP?&s4@g_((4_1L=L$(a91)0=J91Gas#R{McE znYG^9*0A5YZ>#;~+Wkn(W5B0^yELIYLP!K}mB~<)AM@1&nqekynuaEGqPrzoH|KodRXJy)%+w_fu3nE5>@Bd_b zqC$EQ;{c`T&?EsNO|igL9gC7Ygxv?aQUEXMq?~>wg{EyW;VcJ37CUF#HjrT=KQO_* zS>M9yydXk18D(+QDJ1>r);Lav_uYKp$T?4vr{Q$lTo&pKv^?(>L-)G2*lwH!Ah7k? z7oH<8h-(KTKt5V6$8gF)C7Io&P5=SjTh)=zV=E2EUhQZP##L8S{d%UK>>+y82>+FV+#^BzW7u3F)Bb>=lYQ%%j`F>ASe zo*cw@V#u6T`A2He;70mR(V&iV&-7{qP~=SRf&jm9-T{*ZeZ}$rd0#6c&fLG^xJcf5 z+p<`wJYgW+_s*V{uI$nMB;%8`S_3>PfGOj3Rq}@Cx^+j?rk92fANSFDBYnOqQ>Vdj z)(|$AhP4t&Lb=Gvo2#3Gl%9<=Gv`Mz?Po@P4iLF!x}GUWJICDlFk-hS^Whyh7x~VH z@0vD1>HYD4&e+~yzS*-sFR{9`{QEEZO1zg7>R&7cHts-6j!xHVdA8eI+ZlVzd%`es zJT@$#GX(gvCJ1oJN%yLBK}{V=V;seo;!w|Yte!W1%5qLNFWqvZW>h&IiH+oPT=b@E zPhGzv5=(Un*X>v`>%8h_nj^NdYcE6NHS_ifkCV$*D)Tqrbu`s;<=t<4 zAHNqNV?6(g<1PY-w@#I-WYFViz?9TrkMr)u0g`O`u|>T;k|2sV*YF^punvT;$SuTy{j3Gv)yqD!R_CF>yR)MzmmYS5v+~R zXAdD%ng9?df;wd8GxR#%3O+gz};Vo;)sK%Bj-q>Oq%R7JU-KD?vYu>#2UjaDo z&8$>5xW~?KPD_#XFToU1hIb*VOMidUr6iYiO0N|i-7s`T8!cFT`rN!^1Pt78J93i6 z5HI1wIM$94m{3SLDvISDe6$ZG1;eq_D9RTaaC>=cO{@Bs>$IlPCPJJ$h$)-3vzNUQ6OsN#_zWxey!_9%hxwH2_dEJi=yY|1c7nDm2_Lm!Cof8-R_+9UkS zcBE(o47yE)oMR(Q=dp1a2wTX5KvvGyLqlWTa7V&!A*|w|)ax~1_~aJ0=_Lilg*0iQk7#ZD EAHN$8j{pDw literal 0 HcmV?d00001 diff --git a/rd_ui/app/styles/select2.png b/rd_ui/app/styles/select2.png new file mode 100644 index 0000000000000000000000000000000000000000..1d804ffb99699b9e030f1010314de0970b5a000d GIT binary patch literal 613 zcmV-r0-F7aP)#WY!I$JQV$)A5aAS1BM||2XVJl=+L1^1S1H% zM-&lx?NZpUrHhn>fk<>POqf2sh40}xxGZfc+t+#Eb(qHy9_3*1(U%t9t)QDnI#YAL(|ACV(>)>6WD-t!8tutHkdb^#3`HzoJG3A2@T`% zA|K@o*b!`R#(7)PWrMFn2))Ca3MR4(zaT`Zr61*kZK5NPnZwQszxh$fyv3?&4c>$q z2m=+yc0dRXRAsPDxF6sD;@rK4JGdR_``1S~o6Xi@2&aR6hcSrEp9HVRzEqVDqBn<1%hR=D4e1f^ra^A|34Cjc=Gny{F(o#MrvPYgZuTJOz(n)-F<| zj()qR;C={)N<0RRvDZ^@6ND+W*}gh-Lip(MDt!(zMSO)!j2j+*hxgzC-e3$@(O2p* zu;+gddm(cZwXTCLx*Ky4THOa*^b^F`woveIeCK^0aR|TJ00000NkvXXu0mjfA#WC6 literal 0 HcmV?d00001 diff --git a/rd_ui/app/styles/select2x2.png b/rd_ui/app/styles/select2x2.png new file mode 100644 index 0000000000000000000000000000000000000000..4bdd5c961d452c49dfa0789c2c7ffb82c238fc24 GIT binary patch literal 845 zcmV-T1G4;yP)upQ6WKflyv?C|ADVW!U!t`EpA+x zB)5#EjWk-_X77YJZtQo`E0SF)^1bZr%)B7Cd`*OK*r z5WG-7e-R9G9^69ksDt29&oyHqxPSt|-S>xi3%PTd+GjY+BGF|nWC(7D-sd(kxqd9~ zS@2YF5vB+>dP8+$l^{oO3-lEWiGA*QIU)Wds#9M6RZ9N zcQ4y4)xqQOxD=vwu%7cz1nY#$lT&y8HCmkWgpwQP#3dhnYj9|2aS_R}IUF_^6s#$= zTm%~>A#oM?KIg$kh=<`gJkeoHa2LrulVy$Yx+N_0R3$4I!R*0677f(FKqm`2_o4~W z0h}fQZ`lC^1A+m;fM7uI(R1`S0KtG@KrkQ}5DW+&@cTnDVIow56KciMk7a899t0bC zC1KI{TsMe5NAR%GD_5`B-@ad4k~K3SO%H z_M31|`HV?E6)u$E3c&*<*n20+V@mRCop>R5;DWuZCmjSo7p@R&OYl^@G Date: Sun, 13 Apr 2014 16:11:42 +0300 Subject: [PATCH 493/540] Set select2 options via object --- rd_ui/app/scripts/visualizations/base.js | 3 +++ rd_ui/app/views/visualizations/filters.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index f733b1f6e..abc9fa47b 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -68,6 +68,9 @@ template: '\n' + Visualization.renderVisualizationsTemplate, replace: false, link: function (scope) { + scope.select2Options = { + width: '50%' + } scope.$watch('queryResult && queryResult.getFilters()', function (filters) { if (filters) { scope.filters = filters; diff --git a/rd_ui/app/views/visualizations/filters.html b/rd_ui/app/views/visualizations/filters.html index 52fd7079a..9b3226312 100644 --- a/rd_ui/app/views/visualizations/filters.html +++ b/rd_ui/app/views/visualizations/filters.html @@ -1,7 +1,7 @@
        {{filter.friendlyName}}: -
        From 0daf715152589c9d01020038c51151d23e5362c4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 13 Apr 2014 16:35:45 +0300 Subject: [PATCH 494/540] Utility to get last redash release url --- bin/latest_release.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 bin/latest_release.py diff --git a/bin/latest_release.py b/bin/latest_release.py new file mode 100755 index 000000000..ee2debb11 --- /dev/null +++ b/bin/latest_release.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +import sys +import requests + +if __name__ == '__main__': + response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases') + + if response.status_code != 200: + exit("Failed getting releases (status code: %s)." % response.status_code) + + sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True) + + latest_release = sorted_releases[0] + asset_url = latest_release['assets'][0]['url'] + + if '--url-only' in sys.argv: + print asset_url + else: + print "Latest release: %s" % latest_release['tag_name'] + print latest_release['body'] + + print "\nTarball URL: %s" % asset_url + print 'wget: wget --header="Accept: application/octet-stream" %s' % asset_url + + From 6ab4c4551a22b75ba75a5eeb48c8f278dde93fd0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 13 Apr 2014 16:49:57 +0300 Subject: [PATCH 495/540] Fix upload version script. --- bin/upload_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/upload_version.py b/bin/upload_version.py index cb02e4d99..a1d641a76 100644 --- a/bin/upload_version.py +++ b/bin/upload_version.py @@ -20,8 +20,8 @@ if __name__ == '__main__': commit_sha = os.environ['CIRCLE_SHA1'] commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha]) - file_md5_checksum = capture_output(["md5sum", filename]).split()[0] - file_sha256_checksum = capture_output(["sha256sum", filename]).split()[0] + file_md5_checksum = capture_output(["md5sum", filepath]).split()[0] + file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0] version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum) params = json.dumps({ From fec6c8b6a72d2503185a94d54a1301640a2a97d5 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 16 Apr 2014 14:25:40 +0300 Subject: [PATCH 496/540] Fix: selection of true/false values in filters wasn't working. --- rd_ui/app/scripts/services/resources.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 86521e4f2..b7345731b 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -117,7 +117,12 @@ if (!_.isArray(filter.current)) { filter.current = [filter.current]; }; - return (memo && _.some(filter.current, function(v) { return v == row[filter.name] })); + + return (memo && _.some(filter.current, function(v) { + // We compare with either the value or the String representation of the value, + // because Select2 casts true/false to "true"/"false". + return v == row[filter.name] || String(row[filter.name]) == v + })); }, true); }); } else { From efdaf4cf3a09cd32706a4ffbe2166917f92c3f69 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 19 Apr 2014 15:39:09 +0300 Subject: [PATCH 497/540] Typo fix. --- redash/import_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/import_export.py b/redash/import_export.py index 51c0d9cde..67b60106e 100644 --- a/redash/import_export.py +++ b/redash/import_export.py @@ -128,7 +128,7 @@ def importer_with_mapping_file(mapping_filename): def get_data_source(): try: data_source = models.DataSource.get(models.DataSource.name=="Import") - except models.DataSource.DoestNotExist: + except models.DataSource.DoesNotExist: data_source = models.DataSource.create(name="Import", type="import", options='{}') return data_source From 95bcffc28a5f963c1516c3277723b230ffdb0a6c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 19 Apr 2014 15:39:26 +0300 Subject: [PATCH 498/540] Use exec in bin/run. --- bin/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run b/bin/run index eb7f5ff21..cfdee23f8 100755 --- a/bin/run +++ b/bin/run @@ -1,3 +1,3 @@ #!/usr/bin/env bash source .env -"$@" +exec "$@" From 362e5b820e5bd631dc6819356a139089a866c9c4 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 19 Apr 2014 17:06:47 +0300 Subject: [PATCH 499/540] CLI for data sources. --- manage.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/manage.py b/manage.py index a00bdad3c..8badda299 100755 --- a/manage.py +++ b/manage.py @@ -16,6 +16,7 @@ from flask.ext.script import Manager, prompt_pass manager = Manager(app) database_manager = Manager(help="Manages the database (create/drop tables).") users_manager = Manager(help="Users management commands.") +data_sources_manager = Manager(help="Data sources management commands.") @manager.command def version(): @@ -109,9 +110,37 @@ def delete(email): deleted_count = models.User.delete().where(models.User.email == email).execute() print "Deleted %d users." % deleted_count +@data_sources_manager.command +def import_from_settings(name=None): + """Import data source from settings (env variables).""" + name = name or "Default" + data_source = models.DataSource.create(name=name, + type=settings.CONNECTION_ADAPTER, + options=settings.CONNECTION_STRING) + + print "Imported data source from settings (id={}).".format(data_source.id) + + +@data_sources_manager.command +def list(): + """List currently configured data sources""" + for ds in models.DataSource.select(): + print "Name: {}\nType: {}\nOptions: {}".format(ds.name, ds.type, ds.options) + +@data_sources_manager.command +def new(name, type, options): + """Create new data source""" + print "Creating {} data source ({}) with options:\n{}".format(type, name, options) + data_source = models.DataSource.create(name=name, + type=type, + options=options) + print "Id: {}".format(data_source.id) + + manager.add_command("database", database_manager) manager.add_command("users", users_manager) manager.add_command("import", import_manager) +manager.add_command("ds", data_sources_manager) if __name__ == '__main__': manager.run() \ No newline at end of file From 6fe733aeaa99e72709cee838c6aa7b5d1f6fda2a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 20 Apr 2014 09:43:17 +0300 Subject: [PATCH 500/540] Fix: support for .env files without EXPORT --- bin/run | 9 ++++++++- manage.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/run b/bin/run index cfdee23f8..fb0347bcc 100755 --- a/bin/run +++ b/bin/run @@ -1,3 +1,10 @@ #!/usr/bin/env bash -source .env + +# Ideally I would use stdin with source, but in older bash versions this +# wasn't supported properly. +TEMP_ENV_FILE=`mktemp /tmp/redash_env.XXXXXX` +sed 's/^REDASH/export REDASH/' .env > $TEMP_ENV_FILE +source $TEMP_ENV_FILE +rm $TEMP_ENV_FILE + exec "$@" diff --git a/manage.py b/manage.py index 8badda299..ee9ea923d 100755 --- a/manage.py +++ b/manage.py @@ -130,6 +130,7 @@ def list(): @data_sources_manager.command def new(name, type, options): """Create new data source""" + # TODO: validate it's a valid type and in the future, validate the options. print "Creating {} data source ({}) with options:\n{}".format(type, name, options) data_source = models.DataSource.create(name=name, type=type, From 6a7e322b97baa0da4a3f6b9f3309ad4ae9e1c7d0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 22 Apr 2014 16:35:22 +0300 Subject: [PATCH 501/540] Report events at most once per second --- rd_ui/app/scripts/services/services.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/services.js b/rd_ui/app/scripts/services/services.js index 975adc4c4..1e88e6db2 100644 --- a/rd_ui/app/scripts/services/services.js +++ b/rd_ui/app/scripts/services/services.js @@ -20,6 +20,16 @@ } function Events($http) { + this.events = []; + + this.post = _.debounce(function() { + var events = this.events; + this.events = []; + + $http.post('/api/events', events); + + }, 1000); + this.record = function (user, action, object_type, object_id, additional_properties) { var event = { @@ -30,8 +40,9 @@ "timestamp": Date.now()/1000.0 }; _.extend(event, additional_properties); + this.events.push(event); - $http.post('/api/events', [event]); + this.post(); }; } From 976dc1e49699256db3718de8fef5f1bb67b3c391 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 22 Apr 2014 16:35:40 +0300 Subject: [PATCH 502/540] Report event for viewing widget/visualization/query --- rd_ui/app/scripts/controllers/dashboard.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index 34b46a3c7..de388bdc0 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -69,6 +69,9 @@ }; // TODO: fire event for query view for each query + Events.record(currentUser, "view", "widget", $scope.widget.id); + Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id); + Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id); $scope.query = new Query($scope.widget.visualization.query); $scope.queryResult = $scope.query.getQueryResult(); From 73d99031b7a7c092aac84101088d6900987b223e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 25 Apr 2014 17:14:05 +0300 Subject: [PATCH 503/540] Fix #185: when y value is null, convert it to 0. --- rd_ui/app/scripts/services/resources.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index b7345731b..621de4f58 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -150,8 +150,10 @@ xValue = value; point[type] = value; } - if (type == 'y') { + if (value == null) { + value = 0; + } yValues[name] = value; point[type] = value; } From 163ee33ae694554665a5c0637cb227150f318174 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 25 Apr 2014 17:18:39 +0300 Subject: [PATCH 504/540] Fix: job variable was used before assignment --- redash/data/worker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/redash/data/worker.py b/redash/data/worker.py index 2df32a665..583b6bf07 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -278,13 +278,13 @@ class Worker(threading.Thread): logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) _, status = os.waitpid(self.child_pid, 0) self._update_status('done_jobs_count') - if status > 0: - job = Job.load(self.manager.redis_connection, job_id) - if not job.is_finished(): - self._update_status('cancelled_jobs_count') - logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", - self.name, job_id) - job.done(None, "Interrupted/Cancelled while running.") + + job = Job.load(self.manager.redis_connection, job_id) + if status > 0 and not job.is_finished(): + self._update_status('cancelled_jobs_count') + logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", + self.name, job_id) + job.done(None, "Interrupted/Cancelled while running.") job.expire(24 * 3600) From 08d6a904692cb77579292c4b23c957bb7d2a9ecd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 22 Apr 2014 13:37:06 +0300 Subject: [PATCH 505/540] Switch to multiprocessing instead of threading. --- manage.py | 45 ++++++++++++++++++++++++------------------ redash/data/manager.py | 5 ++++- redash/data/worker.py | 38 ++++++++++++++++++++++++++++------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/manage.py b/manage.py index ee9ea923d..0d2a6fb05 100755 --- a/manage.py +++ b/manage.py @@ -3,6 +3,8 @@ CLI to manage redash. """ import atfork +import signal + atfork.monkeypatch_os_fork_functions() import atfork.stdlib_fixer atfork.stdlib_fixer.fix_logging_module() @@ -28,26 +30,31 @@ def version(): def runworkers(): """Starts the re:dash query executors/workers.""" - try: - old_workers = data_manager.redis_connection.smembers('workers') - data_manager.redis_connection.delete('workers') - - logging.info("Cleaning old workers: %s", old_workers) - - data_manager.start_workers(settings.WORKERS_COUNT) - logging.info("Workers started.") - - while True: - try: - data_manager.refresh_queries() - data_manager.report_status() - except Exception as e: - logging.error("Something went wrong with refreshing queries...") - logging.exception(e) - time.sleep(60) - except KeyboardInterrupt: - logging.warning("Exiting; waiting for threads") + def stop_handler(signum, frame): + logging.warning("Exiting; waiting for workers") data_manager.stop_workers() + exit() + + signal.signal(signal.SIGTERM, stop_handler) + signal.signal(signal.SIGINT, stop_handler) + + old_workers = data_manager.redis_connection.smembers('workers') + data_manager.redis_connection.delete('workers') + + logging.info("Cleaning old workers: %s", old_workers) + + data_manager.start_workers(settings.WORKERS_COUNT) + logging.info("Workers started.") + + while True: + try: + data_manager.refresh_queries() + data_manager.report_status() + except Exception as e: + logging.error("Something went wrong with refreshing queries...") + logging.exception(e) + time.sleep(60) + @manager.shell def make_shell_context(): diff --git a/redash/data/manager.py b/redash/data/manager.py index 3b63a9cfe..225ed1994 100755 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -142,6 +142,7 @@ class Manager(object): redis_connection_params = self.redis_connection.connection_pool.connection_kwargs self.workers = [worker.Worker(worker_id, self, redis_connection_params) for worker_id in xrange(workers_count)] + for w in self.workers: w.start() @@ -149,7 +150,9 @@ class Manager(object): def stop_workers(self): for w in self.workers: - w.continue_working = False + w.terminate() + + for w in self.workers: w.join() def _save_status(self): diff --git a/redash/data/worker.py b/redash/data/worker.py index 583b6bf07..430c635b7 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -3,8 +3,8 @@ Worker implementation to execute incoming queries. """ import json import logging +import multiprocessing import os -import threading import uuid import datetime import time @@ -209,31 +209,30 @@ class Job(RedisObject): return "" % (self.id, self.priority, self.status) -class Worker(threading.Thread): +class Worker(multiprocessing.Process): def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1): self.manager = manager self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) + self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() if k in ('host', 'db', 'password', 'port')} + self.worker_id = None self.continue_working = True self.sleep_time = sleep_time self.child_pid = None - self.worker_id = worker_id + self.current_job_id = None self.status = { - 'id': self.worker_id, 'jobs_count': 0, 'cancelled_jobs_count': 0, 'done_jobs_count': 0, 'updated_at': time.time(), 'started_at': time.time() } - self._save_status() - self.manager.redis_connection.sadd('workers', self._key) - super(Worker, self).__init__(name="Worker-%s" % self.worker_id) + super(Worker, self).__init__(name="Worker") def set_title(self, title=None): base_title = "redash worker:%s" % self.worker_id @@ -245,7 +244,28 @@ class Worker(threading.Thread): setproctitle.setproctitle(full_title) def run(self): + self.worker_id = os.getpid() + self.status['id'] = self.worker_id + self.name = "Worker:%d" % self.worker_id + self.manager.redis_connection.sadd('workers', self._key) + self._save_status() + self.set_title() + logging.info("[%s] started.", self.name) + + signal.signal(signal.SIGINT, self._stop) + signal.signal(signal.SIGTERM, self._stop) + + self._wait_for_jobs() + + def _stop(self, signum, frame): + self.continue_working = False + if self.current_job_id: + job = Job.load(self.manager.redis_connection, self.current_job_id) + if job: + job.cancel() + + def _wait_for_jobs(self): while self.continue_working: job_id = self.manager.queue.pop() if job_id: @@ -270,6 +290,7 @@ class Worker(threading.Thread): self.manager.redis_connection.hmset(self._key, self.status) def _fork_and_process(self, job_id): + self.current_job_id = job_id self.child_pid = os.fork() if self.child_pid == 0: self.set_title("processing %s" % job_id) @@ -291,6 +312,9 @@ class Worker(threading.Thread): logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) + self.child_pid = None + self.current_job_id = None + def _process(self, job_id): redis_connection = redis.StrictRedis(**self.redis_connection_params) job = Job.load(redis_connection, job_id) From 413dd61491516bf9866f301083fce525e55c5003 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 22 Apr 2014 13:37:48 +0300 Subject: [PATCH 506/540] Remove atfork (it's not needed anymore) --- manage.py | 6 ------ requirements.txt | 1 - 2 files changed, 7 deletions(-) diff --git a/manage.py b/manage.py index 0d2a6fb05..6640d1718 100755 --- a/manage.py +++ b/manage.py @@ -2,13 +2,7 @@ """ CLI to manage redash. """ -import atfork import signal - -atfork.monkeypatch_os_fork_functions() -import atfork.stdlib_fixer -atfork.stdlib_fixer.fix_logging_module() - import logging import time from redash import settings, app, db, models, data_manager, __version__ diff --git a/requirements.txt b/requirements.txt index 39b7e291c..6a11f490b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ MarkupSafe==0.18 WTForms==1.0.5 Werkzeug==0.9.4 aniso8601==0.82 -atfork==0.1.2 blinker==1.3 flask-peewee==0.6.5 itsdangerous==0.23 From 2938e579802dd2c79bdb40d82f07e4aa187aece9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 22 Apr 2014 14:53:08 +0300 Subject: [PATCH 507/540] Cleaner shutdown (#8) --- redash/data/query_runner_pg.py | 27 +++++++++++++++++---------- redash/data/worker.py | 9 ++++++++- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/redash/data/query_runner_pg.py b/redash/data/query_runner_pg.py index 945de3872..487cf3268 100644 --- a/redash/data/query_runner_pg.py +++ b/redash/data/query_runner_pg.py @@ -8,7 +8,7 @@ query language (for example: HiveQL). import json import sys import select - +import logging import psycopg2 from redash.utils import JSONEncoder @@ -20,15 +20,18 @@ def pg(connection_string): def wait(conn): while 1: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) + try: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + except select.error: + raise psycopg2.OperationalError("select.error received") def query_runner(query): connection = psycopg2.connect(connection_string, async=True) @@ -51,6 +54,10 @@ def pg(connection_string): json_data = json.dumps(data, cls=JSONEncoder) error = None cursor.close() + except (select.error, OSError, psycopg2.OperationalError) as e: + logging.exception(e) + error = "Query interrupted. Please retry." + json_data = None except psycopg2.DatabaseError as e: json_data = None error = e.message diff --git a/redash/data/worker.py b/redash/data/worker.py index 430c635b7..fcbbfd774 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -297,7 +297,14 @@ class Worker(multiprocessing.Process): self._process(job_id) else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) - _, status = os.waitpid(self.child_pid, 0) + + try: + _, status = os.waitpid(self.child_pid, 0) + except OSError: + logging.info("[%s] OSError while waiting for child to finish", self.name) + # setting status to >0, so the job cleanup is triggered + status = 1 + self._update_status('done_jobs_count') job = Job.load(self.manager.redis_connection, job_id) From 084e9f83942d7d02ad1743e6787f26119b244ce7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 10:35:47 +0300 Subject: [PATCH 508/540] Migration to add text column to widgets and make visualization_id nullable. --- migrations/add_text_to_widgets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 migrations/add_text_to_widgets.py diff --git a/migrations/add_text_to_widgets.py b/migrations/add_text_to_widgets.py new file mode 100644 index 000000000..50a40fd91 --- /dev/null +++ b/migrations/add_text_to_widgets.py @@ -0,0 +1,13 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.add_column(models.Widget, models.Widget.text, 'text') + migrator.set_nullable(models.Widget, models.Widget.visualization, True) + + db.close_db(None) \ No newline at end of file From fa2986a154401e518b916a5dae4a4d97110edd88 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 10:36:00 +0300 Subject: [PATCH 509/540] Add underscore.string lib. --- rd_ui/app/index.html | 2 +- rd_ui/bower.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 67cac81dd..e1c4f4804 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -109,7 +109,7 @@ - + diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 5f9c86d88..e95631a33 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -20,7 +20,8 @@ "cornelius": "https://github.com/restorando/cornelius.git", "gridster": "0.2.0", "mousetrap": "~1.4.6", - "angular-ui-select2": "~0.0.5" + "angular-ui-select2": "~0.0.5", + "underscore.string": "~2.3.3" }, "devDependencies": { "angular-mocks": "~1.0.7", From b7a0b7454ad04fd38ad1917cd5816f63c1539196 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 10:36:56 +0300 Subject: [PATCH 510/540] Add textbox widget support. --- rd_ui/app/scripts/controllers/dashboard.js | 41 +++++++------- .../directives/dashboard_directives.js | 36 +++++++++---- rd_ui/app/scripts/services/resources.js | 7 +++ rd_ui/app/views/dashboard.html | 11 +++- rd_ui/app/views/new_widget_form.html | 54 ++++++++++++------- redash/models.py | 23 ++++---- tests/test_controllers.py | 17 ++++++ 7 files changed, 130 insertions(+), 59 deletions(-) diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index de388bdc0..878b5f3b4 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -1,13 +1,16 @@ (function() { - var DashboardCtrl = function($scope, Events, $routeParams, $http, $timeout, Dashboard) { + var DashboardCtrl = function($scope, Events, Widget, $routeParams, $http, $timeout, Dashboard) { Events.record(currentUser, "view", "dashboard", dashboard.id); $scope.refreshEnabled = false; $scope.refreshRate = 60; - $scope.dashboard = Dashboard.get({ - slug: $routeParams.dashboardSlug - }, function(dashboard) { + $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) { $scope.$parent.pageTitle = dashboard.name; + $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) { + return _.map(row, function (widget) { + return new Widget(widget); + }); + }); }); var autoRefresh = function() { @@ -51,39 +54,41 @@ }; }; - var WidgetCtrl = function($scope, Events, $http, $location, Query) { + var WidgetCtrl = function($scope, Events, Widget, $http, $location, Query) { $scope.deleteWidget = function() { - if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) { + if (!confirm('Are you sure you want to remove "' + $scope.widget.getName() + '" from the dashboard?')) { return; } Events.record(currentUser, "delete", "widget", $scope.widget.id); - $http.delete('/api/widgets/' + $scope.widget.id).success(function() { + $scope.widget.$delete(function() { $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) { return _.filter(row, function(widget) { - return widget.id != $scope.widget.id; + return widget.id != undefined; }) }); }); }; - // TODO: fire event for query view for each query Events.record(currentUser, "view", "widget", $scope.widget.id); - Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id); - Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id); - $scope.query = new Query($scope.widget.visualization.query); - $scope.queryResult = $scope.query.getQueryResult(); + if ($scope.widget.visualization) { + Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id); + Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id); - $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString(); - $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow(); + $scope.query = new Query($scope.widget.visualization.query); + $scope.queryResult = $scope.query.getQueryResult(); + $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow(); - $scope.updateTime = ''; + $scope.type = 'visualization'; + } else { + $scope.type = 'textbox'; + } }; angular.module('redash.controllers') - .controller('DashboardCtrl', ['$scope', 'Events', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) - .controller('WidgetCtrl', ['$scope', 'Events', '$http', '$location', 'Query', WidgetCtrl]) + .controller('DashboardCtrl', ['$scope', 'Events', 'Widget', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) + .controller('WidgetCtrl', ['$scope', 'Events', 'Widget', '$http', '$location', 'Query', WidgetCtrl]) })(); \ No newline at end of file diff --git a/rd_ui/app/scripts/directives/dashboard_directives.js b/rd_ui/app/scripts/directives/dashboard_directives.js index c1bc2dde3..368907074 100644 --- a/rd_ui/app/scripts/directives/dashboard_directives.js +++ b/rd_ui/app/scripts/directives/dashboard_directives.js @@ -46,7 +46,7 @@ row: rowIndex + 1, ySize: 1, xSize: widget.width, - name: widget.visualization.query.name + name: widget.getName()//visualization.query.name }); }); }); @@ -125,25 +125,37 @@ value: 2 }]; + $scope.type = 'visualization'; + + $scope.isVisualization = function () { + return $scope.type == 'visualization'; + }; + + $scope.isTextBox = function () { + return $scope.type == 'textbox'; + }; + + $scope.setType = function (type) { + $scope.type = type; + }; + var reset = function() { $scope.saveInProgress = false; $scope.widgetSize = 1; $scope.queryId = null; $scope.selectedVis = null; $scope.query = null; - - } + $scope.text = ""; + }; reset(); - $scope.loadVisualizations = function() { + $scope.loadVisualizations = function () { if (!$scope.queryId) { return; } - Query.get({ - id: $scope.queryId - }, function(query) { + Query.get({ id: $scope.queryId }, function(query) { if (query) { $scope.query = query; if (query.visualizations.length) { @@ -157,19 +169,21 @@ $scope.saveInProgress = true; var widget = new Widget({ - 'visualization_id': $scope.selectedVis.id, + 'visualization_id': $scope.selectedVis && $scope.selectedVis.id, 'dashboard_id': $scope.dashboard.id, 'options': {}, - 'width': $scope.widgetSize + 'width': $scope.widgetSize, + 'text': $scope.text }); widget.$save().then(function(response) { // update dashboard layout $scope.dashboard.layout = response['layout']; + var newWidget = new Widget(response['widget']); if (response['new_row']) { - $scope.dashboard.widgets.push([response['widget']]); + $scope.dashboard.widgets.push([newWidget]); } else { - $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']); + $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(newWidget); } // close the dialog diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 621de4f58..429f9c61a 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -363,6 +363,13 @@ var Widget = function ($resource) { var WidgetResource = $resource('/api/widgets/:id', {id: '@id'}); + WidgetResource.prototype.getName = function () { + if (this.visualization) { + return this.visualization.query.name + ' (' + this.visualization.name + ')'; + } + return _.str.truncate(this.text, 20); + }; + return WidgetResource; } diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index d881680ac..624187cbe 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -21,7 +21,7 @@
        -
        +

        @@ -43,7 +43,16 @@ +

        +
        +
        +
        + {{widget.text}} + + + +
        diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 10b41b43c..395babf3d 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -6,32 +6,46 @@

        Add Widget

        -

        -

        -
        - -
        - -
        +

        + +

        -
        -
        - - -
        +
        + + Markdown supported. +
        -
        - - -
        +
        +

        +

        +
        + +
        + +
        +

        + +
        +
        + + +
        +
        +
        + +
        + +
        -
        + +
        - +
        diff --git a/redash/models.py b/redash/models.py index 44c313819..a79c48f46 100644 --- a/redash/models.py +++ b/redash/models.py @@ -255,11 +255,11 @@ class Dashboard(BaseModel): if with_widgets: widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\ .where(Widget.dashboard == self.id)\ - .join(Visualization)\ - .join(Query)\ - .join(User)\ + .join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\ + .join(Query, join_type=peewee.JOIN_LEFT_OUTER)\ + .join(User, join_type=peewee.JOIN_LEFT_OUTER)\ .switch(Query)\ - .join(QueryResult) + .join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER) widgets = {w.id: w.to_dict() for w in widgets} widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) else: @@ -324,8 +324,8 @@ class Visualization(BaseModel): class Widget(BaseModel): id = peewee.PrimaryKeyField() - visualization = peewee.ForeignKeyField(Visualization, related_name='widgets') - + visualization = peewee.ForeignKeyField(Visualization, related_name='widgets', null=True) + text = peewee.TextField(null=True) width = peewee.IntegerField() options = peewee.TextField() dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True) @@ -339,14 +339,19 @@ class Widget(BaseModel): db_table = 'widgets' def to_dict(self): - return { + d = { 'id': self.id, 'width': self.width, 'options': json.loads(self.options), - 'visualization': self.visualization.to_dict(), - 'dashboard_id': self._data['dashboard'] + 'dashboard_id': self._data['dashboard'], + 'text': self.text } + if self.visualization and self.visualization.id: + d['visualization'] = self.visualization.to_dict() + + return d + def __unicode__(self): return u"%s" % self.id diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 31412991e..7bdd09d6f 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -182,6 +182,23 @@ class WidgetAPITest(BaseTestCase): [rv4.json['widget']['id']]]) self.assertEquals(rv4.json['new_row'], True) + def test_create_text_widget(self): + dashboard = dashboard_factory.create() + + data = { + 'visualization_id': None, + 'text': 'Sample text.', + 'dashboard_id': dashboard.id, + 'options': {}, + 'width': 2 + } + + with app.test_client() as c, authenticated_user(c): + rv = json_request(c.post, '/api/widgets', data=data) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.json['widget']['text'], 'Sample text.') + def test_delete_widget(self): widget = widget_factory.create() From 2caf02b4e00e3cea94101de6a9507d1dc7f30037 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 11:15:19 +0300 Subject: [PATCH 511/540] Markdown support for textbox. --- rd_ui/app/index.html | 2 +- rd_ui/app/scripts/filters.js | 114 ++++++++++++++------------- rd_ui/app/views/dashboard.html | 2 +- rd_ui/app/views/new_widget_form.html | 11 ++- rd_ui/bower.json | 3 +- 5 files changed, 72 insertions(+), 60 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index e1c4f4804..8f5cdcbeb 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -110,10 +110,10 @@ + - diff --git a/rd_ui/app/scripts/filters.js b/rd_ui/app/scripts/filters.js index 0ecb0e4e6..4e3665e03 100644 --- a/rd_ui/app/scripts/filters.js +++ b/rd_ui/app/scripts/filters.js @@ -1,61 +1,67 @@ var durationHumanize = function (duration) { - var humanized = ""; - if (duration == undefined) { - humanized = "-"; - } else if (duration < 60) { - humanized = Math.round(duration) + "s"; - } else if (duration > 3600*24) { - var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0); - humanized = days + "days"; - } else if (duration >= 3600) { - var hours = Math.round(parseFloat(duration) / 60.0 / 60.0); - humanized = hours + "h"; - } else { - var minutes = Math.round(parseFloat(duration) / 60.0); - humanized = minutes + "m"; - } - return humanized; + var humanized = ""; + if (duration == undefined) { + humanized = "-"; + } else if (duration < 60) { + humanized = Math.round(duration) + "s"; + } else if (duration > 3600 * 24) { + var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0); + humanized = days + "days"; + } else if (duration >= 3600) { + var hours = Math.round(parseFloat(duration) / 60.0 / 60.0); + humanized = hours + "h"; + } else { + var minutes = Math.round(parseFloat(duration) / 60.0); + humanized = minutes + "m"; + } + return humanized; } angular.module('redash.filters', []). - filter('durationHumanize', function () { - return durationHumanize; - }) + filter('durationHumanize', function () { + return durationHumanize; + }) - .filter('refreshRateHumanize', function () { - return function (ttl) { - if (ttl==-1) { - return "Never"; - } else { - return "Every " + durationHumanize(ttl); - } - } - }) + .filter('refreshRateHumanize', function () { + return function (ttl) { + if (ttl == -1) { + return "Never"; + } else { + return "Every " + durationHumanize(ttl); + } + } + }) - .filter('toHuman', function() { - return function(text) { - return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { - return a.toUpperCase(); - }); - } - }) + .filter('toHuman', function () { + return function (text) { + return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { + return a.toUpperCase(); + }); + } + }) - .filter('colWidth', function () { - return function (widgetWidth) { - if (widgetWidth == 1) { - return 6; - } - return 12; - } - }) - - .filter('capitalize', function () { - return function (text) { - if (text) { - return text[0].toUpperCase() + text.slice(1).toLowerCase(); - } else { - return null; - } - - } - }); \ No newline at end of file + .filter('colWidth', function () { + return function (widgetWidth) { + if (widgetWidth == 1) { + return 6; + } + return 12; + } + }) + + .filter('capitalize', function () { + return function (text) { + if (text) { + return _.str.capitalize(text); + } else { + return null; + } + + } + }) + + .filter('markdown', ['$sce', function($sce) { + return function(text) { + return $sce.trustAsHtml(marked(text)); + } + }]); \ No newline at end of file diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index 624187cbe..b50b6559f 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -48,7 +48,7 @@
        - {{widget.text}} +

        diff --git a/rd_ui/app/views/new_widget_form.html b/rd_ui/app/views/new_widget_form.html index 395babf3d..9ebb9f8fc 100644 --- a/rd_ui/app/views/new_widget_form.html +++ b/rd_ui/app/views/new_widget_form.html @@ -11,9 +11,14 @@

        -
        - - Markdown supported. +
        +
        + +
        +
        + Preview: +

        +
        diff --git a/rd_ui/bower.json b/rd_ui/bower.json index e95631a33..830255540 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -21,7 +21,8 @@ "gridster": "0.2.0", "mousetrap": "~1.4.6", "angular-ui-select2": "~0.0.5", - "underscore.string": "~2.3.3" + "underscore.string": "~2.3.3", + "marked": "~0.3.2" }, "devDependencies": { "angular-mocks": "~1.0.7", From ef366df1fb4492194e84daf6eedf351f2fc2e0b1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 11:23:32 +0300 Subject: [PATCH 512/540] Remove unused dependencies --- rd_ui/app/scripts/controllers/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index 878b5f3b4..b1daba4dd 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -54,7 +54,7 @@ }; }; - var WidgetCtrl = function($scope, Events, Widget, $http, $location, Query) { + var WidgetCtrl = function($scope, Events, Query) { $scope.deleteWidget = function() { if (!confirm('Are you sure you want to remove "' + $scope.widget.getName() + '" from the dashboard?')) { return; @@ -89,6 +89,6 @@ angular.module('redash.controllers') .controller('DashboardCtrl', ['$scope', 'Events', 'Widget', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl]) - .controller('WidgetCtrl', ['$scope', 'Events', 'Widget', '$http', '$location', 'Query', WidgetCtrl]) + .controller('WidgetCtrl', ['$scope', 'Events', 'Query', WidgetCtrl]) })(); \ No newline at end of file From e5bf43198716b5cb117c5d0be2191450b4e3651b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 11:37:42 +0300 Subject: [PATCH 513/540] Fix: Chart type resets to Date/Time when editing #186 --- rd_ui/app/scripts/visualizations/chart.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rd_ui/app/scripts/visualizations/chart.js b/rd_ui/app/scripts/visualizations/chart.js index e0208d358..096839988 100644 --- a/rd_ui/app/scripts/visualizations/chart.js +++ b/rd_ui/app/scripts/visualizations/chart.js @@ -101,6 +101,8 @@ } }); + scope.xAxisType = (scope.visualization.options.xAxis && scope.visualization.options.xAxis.type) || scope.xAxisType; + xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) { scope.visualization.options.xAxis = scope.visualization.options.xAxis || {}; scope.visualization.options.xAxis.type = xAxisType; From 391c2206042283866b898a366bdcf4c2f715ba7f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 11:57:16 +0300 Subject: [PATCH 514/540] Show error message if failed deleting a visualization --- rd_ui/app/scripts/controllers/query_source.js | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_source.js b/rd_ui/app/scripts/controllers/query_source.js index 075b9dee9..a104e365e 100644 --- a/rd_ui/app/scripts/controllers/query_source.js +++ b/rd_ui/app/scripts/controllers/query_source.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - function QuerySourceCtrl(Events, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) { + function QuerySourceCtrl(Events, growl, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) { // extends QueryViewCtrl $controller('QueryViewCtrl', {$scope: $scope}); // TODO: @@ -67,15 +67,19 @@ if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) { Events.record(currentUser, 'delete', 'visualization', vis.id); - Visualization.delete(vis); - if ($scope.selectedTab == vis.id) { - $scope.selectedTab = DEFAULT_TAB; - $location.hash($scope.selectedTab); - } - $scope.query.visualizations = - $scope.query.visualizations.filter(function(v) { - return vis.id !== v.id; - }); + Visualization.delete(vis, function() { + if ($scope.selectedTab == vis.id) { + $scope.selectedTab = DEFAULT_TAB; + $location.hash($scope.selectedTab); + } + $scope.query.visualizations = + $scope.query.visualizations.filter(function (v) { + return vis.id !== v.id; + }); + }, function () { + growl.addErrorMessage("Error deleting visualization. Maybe it's used in a dashboard?"); + }); + } }; @@ -99,7 +103,7 @@ } angular.module('redash.controllers').controller('QuerySourceCtrl', [ - 'Events', '$controller', '$scope', '$location', 'Query', + 'Events', 'growl', '$controller', '$scope', '$location', 'Query', 'Visualization', 'KeyboardShortcuts', QuerySourceCtrl ]); })(); \ No newline at end of file From a71b99a873f3e132d36dca07f45d89c19a98c0eb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 12:09:38 +0300 Subject: [PATCH 515/540] Workaround for cases when widget is missing but referenced in a dashboard layout (re. #120) --- redash/models.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index a79c48f46..071502012 100644 --- a/redash/models.py +++ b/redash/models.py @@ -261,7 +261,23 @@ class Dashboard(BaseModel): .switch(Query)\ .join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER) widgets = {w.id: w.to_dict() for w in widgets} - widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) + + # The following is a workaround for cases when the widget object gets deleted without the dashboard layout + # updated. This happens for users with old databases that didn't have a foreign key relationship between + # visualizations and widgets. + # It's temporary until better solution is implemented (we probably should move the position information + # to the widget). + widgets_layout = [] + for row in layout: + new_row = [] + for widget_id in row: + widget = widgets.get(widget_id, None) + if widget: + new_row.append(widget) + + widgets_layout.append(new_row) + + # widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout) else: widgets_layout = None From a8ea811fedfd39217ba0b34b32d89b1f44c02e44 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 12:13:33 +0300 Subject: [PATCH 516/540] Make job expiry time configurable. --- redash/data/worker.py | 2 +- redash/settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/redash/data/worker.py b/redash/data/worker.py index fcbbfd774..5b2fbdafd 100644 --- a/redash/data/worker.py +++ b/redash/data/worker.py @@ -314,7 +314,7 @@ class Worker(multiprocessing.Process): self.name, job_id) job.done(None, "Interrupted/Cancelled while running.") - job.expire(24 * 3600) + job.expire(settings.JOB_EXPIRY_TIME) logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) diff --git a/redash/settings.py b/redash/settings.py index a225c1206..50421354c 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -63,6 +63,7 @@ ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", '')) ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", '')) STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/")) WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2")) +JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*24)) COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "") From a7ce5246a645b29dc95166cdf401dc3925709a5f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 29 Apr 2014 16:02:10 +0300 Subject: [PATCH 517/540] Fix: return last cached result for ttl=-1 (fix #174) --- redash/models.py | 9 +++++++-- tests/test_models.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/redash/models.py b/redash/models.py index 071502012..bdc061e07 100644 --- a/redash/models.py +++ b/redash/models.py @@ -133,8 +133,13 @@ class QueryResult(BaseModel): def get_latest(cls, data_source, query, ttl=0): query_hash = utils.gen_query_hash(query) - query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source, - peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc()) + if ttl == -1: + query = cls.select().where(cls.query_hash == query_hash, + cls.data_source == data_source).order_by(cls.retrieved_at.desc()) + else: + query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source, + peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", + ttl)).order_by(cls.retrieved_at.desc()) return query.first() diff --git a/tests/test_models.py b/tests/test_models.py index 854929c5b..4ec0f10e6 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -80,4 +80,14 @@ class QueryResultTest(BaseTestCase): found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60) + self.assertEqual(found_query_result.id, qr.id) + + def test_get_latest_returns_the_last_cached_result_for_negative_ttl(self): + yesterday = datetime.datetime.now() + datetime.timedelta(days=-100) + very_old = query_result_factory.create(retrieved_at=yesterday) + + yesterday = datetime.datetime.now() + datetime.timedelta(days=-1) + qr = query_result_factory.create(retrieved_at=yesterday) + found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, -1) + self.assertEqual(found_query_result.id, qr.id) \ No newline at end of file From 6ae3a7552ae8f833babc567fb946315d42a8c93f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 1 May 2014 17:45:35 +0300 Subject: [PATCH 518/540] Fix #91: better filtering of bad tokens in column names --- rd_ui/app/scripts/services/resources.js | 30 +++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 429f9c61a..9331be375 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -205,19 +205,35 @@ return this.columns; } - QueryResult.prototype.getColumnCleanName = function (column) { + + QueryResult.prototype.getColumnNameWithoutType = function (column) { var parts = column.split('::'); - var name = parts[1]; - if (parts[0] != '') { - // TODO: it's probably time to generalize this. - // see also getColumnFriendlyName - name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g, ''); + return parts[0]; + }; + + var charConversionMap = { + '__pct': /%/g, + '_': / /g, + '__qm': /\?/g, + '__brkt': /[\(\)\[\]]/g, + '__dash': /-/g, + '__amp': /&/g + }; + + QueryResult.prototype.getColumnCleanName = function (column) { + var name = this.getColumnNameWithoutType(column); + + if (name != '') { + _.each(charConversionMap, function(regex, replacement) { + name = name.replace(regex, replacement); + }); } + return name; } QueryResult.prototype.getColumnFriendlyName = function (column) { - return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) { + return this.getColumnNameWithoutType(column).replace(/(?:^|\s)\S/g, function (a) { return a.toUpperCase(); }); } From 9a6852db78a3d2419ec574584e52ebcd3644a791 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 1 May 2014 17:52:42 +0300 Subject: [PATCH 519/540] Fix #195: When two columns have the same name their values get overriden --- redash/data/query_runner_pg.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/redash/data/query_runner_pg.py b/redash/data/query_runner_pg.py index 487cf3268..7facb76ce 100644 --- a/redash/data/query_runner_pg.py +++ b/redash/data/query_runner_pg.py @@ -43,12 +43,26 @@ def pg(connection_string): cursor.execute(query) wait(connection) - column_names = [col.name for col in cursor.description] + column_names = set() + columns = [] + duplicates_counter = 1 + + for column in cursor.description: + # TODO: this deduplication needs to be generalized and reused in all query runners. + column_name = column.name + if column_name in column_names: + column_name = column_name + str(duplicates_counter) + duplicates_counter += 1 + + column_names.add(column_name) + + columns.append({ + 'name': column_name, + 'friendly_name': column_friendly_name(column_name), + 'type': None + }) rows = [dict(zip(column_names, row)) for row in cursor] - columns = [{'name': col.name, - 'friendly_name': column_friendly_name(col.name), - 'type': None} for col in cursor.description] data = {'columns': columns, 'rows': rows} json_data = json.dumps(data, cls=JSONEncoder) From 79b4c86520d15f59ce2f6183d0dd7534ced9f452 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 13:14:49 +0300 Subject: [PATCH 520/540] Improve latest_release utility --- bin/latest_release.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/latest_release.py b/bin/latest_release.py index ee2debb11..1c24f6815 100755 --- a/bin/latest_release.py +++ b/bin/latest_release.py @@ -12,14 +12,19 @@ if __name__ == '__main__': latest_release = sorted_releases[0] asset_url = latest_release['assets'][0]['url'] + filename = latest_release['assets'][0]['name'] + + wget_command = 'wget --header="Accept: application/octet-stream" %s -O %s' % (asset_url, filename) if '--url-only' in sys.argv: print asset_url + elif '--wget' in sys.argv: + print wget_command else: print "Latest release: %s" % latest_release['tag_name'] print latest_release['body'] print "\nTarball URL: %s" % asset_url - print 'wget: wget --header="Accept: application/octet-stream" %s' % asset_url + print 'wget: %s' % (wget_command) From 1bf56899f3d454b9819089257072339739efa211 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 13:23:18 +0300 Subject: [PATCH 521/540] Open visualization editor when adding a new visualization. --- rd_ui/app/scripts/visualizations/base.js | 1 + rd_ui/app/views/query.html | 2 +- rd_ui/app/views/visualizations/edit_visualization.html | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/base.js b/rd_ui/app/scripts/visualizations/base.js index abc9fa47b..fdea79294 100644 --- a/rd_ui/app/scripts/visualizations/base.js +++ b/rd_ui/app/scripts/visualizations/base.js @@ -104,6 +104,7 @@ query: '=', queryResult: '=', visualization: '=?', + openEditor: '=?', onNewSuccess: '=?' }, link: function (scope, element, attrs) { diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 926a9ff2b..5142b7088 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -172,7 +172,7 @@
        - +
        diff --git a/rd_ui/app/views/visualizations/edit_visualization.html b/rd_ui/app/views/visualizations/edit_visualization.html index 32bace4ab..a285865ef 100644 --- a/rd_ui/app/views/visualizations/edit_visualization.html +++ b/rd_ui/app/views/visualizations/edit_visualization.html @@ -1,7 +1,7 @@
        - Edit + Edit -
        +
        From 96081de51f3354aa5fc19907cec170c27a701e61 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 13:46:11 +0300 Subject: [PATCH 522/540] update makefile to set version" --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4e44ebb23..61989e15f 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ deps: cd rd_ui && grunt build pack: + sed -r "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py > redash/__init__.py tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: From 9a025a7e056acdfaf0b071c317e1422a20b24de5 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 14:02:20 +0300 Subject: [PATCH 523/540] Fix the pack make command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 61989e15f..2358c7051 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ deps: cd rd_ui && grunt build pack: - sed -r "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py > redash/__init__.py + sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: From ab2fa1e3529bf4b2b93fcaae18800466daa98f85 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 14:25:20 +0300 Subject: [PATCH 524/540] Show version in admin/status --- redash/controllers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index ceb140412..e9195235a 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -19,7 +19,7 @@ from flask_login import current_user, login_user, logout_user import sqlparse import events from permissions import require_permission -from redash import settings, utils +from redash import settings, utils, __version__ from redash import data from redash import app, auth, api, redis_connection, data_manager @@ -93,7 +93,7 @@ def status_api(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] - + status['version'] = __version__ status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['dashboards_count'] = models.Dashboard.select().count() From 174eb2408e0ddf1115ea08743859dc6b3837d500 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 14:37:20 +0300 Subject: [PATCH 525/540] Fix version name in tarball --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2358c7051..8920c86d8 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ NAME=redash VERSION=`python ./manage.py version` FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM) -FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz +# VERSION gets evaluated every time it's referenced, therefore we need to use VERSION here instead of FULL_VERSION. +FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).tar.gz deps: cd rd_ui && npm install @@ -14,7 +15,7 @@ pack: tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: - python bin/upload_version.py $(FULL_VERSION) $(FILENAME) + python bin/upload_version.py $(VERSION) $(FILENAME) test: nosetests --with-coverage --cover-package=redash tests/*.py From a7b10db3f4d6356b7604dbc5683f048e2d75cdad Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sun, 4 May 2014 15:58:28 +0300 Subject: [PATCH 526/540] Fix: set was messing up column order --- redash/data/query_runner_pg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/redash/data/query_runner_pg.py b/redash/data/query_runner_pg.py index 7facb76ce..cf094e461 100644 --- a/redash/data/query_runner_pg.py +++ b/redash/data/query_runner_pg.py @@ -43,7 +43,9 @@ def pg(connection_string): cursor.execute(query) wait(connection) - column_names = set() + # While set would be more efficient here, it sorts the data which is not what we want, but due to the small + # size of the data we can assume it's ok. + column_names = [] columns = [] duplicates_counter = 1 @@ -54,7 +56,7 @@ def pg(connection_string): column_name = column_name + str(duplicates_counter) duplicates_counter += 1 - column_names.add(column_name) + column_names.append(column_name) columns.append({ 'name': column_name, From f95a09a015023edeaa7b99d8d7cba96b439bf004 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:34:29 +0300 Subject: [PATCH 527/540] Widget: accessor function to get Query object. --- rd_ui/app/scripts/services/resources.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 9331be375..b97e7c570 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -376,9 +376,17 @@ return DataSourceResource; } - var Widget = function ($resource) { + var Widget = function ($resource, Query) { var WidgetResource = $resource('/api/widgets/:id', {id: '@id'}); + WidgetResource.prototype.getQuery = function () { + if (!this.query && this.visualization) { + this.query = new Query(this.visualization.query); + } + + return this.query; + }; + WidgetResource.prototype.getName = function () { if (this.visualization) { return this.visualization.query.name + ' (' + this.visualization.name + ')'; @@ -393,5 +401,5 @@ .factory('QueryResult', ['$resource', '$timeout', QueryResult]) .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query]) .factory('DataSource', ['$resource', DataSource]) - .factory('Widget', ['$resource', Widget]); + .factory('Widget', ['$resource', 'Query', Widget]); })(); From 638fb123ece62c907b1f0886c9600a23df8cc51a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:34:54 +0300 Subject: [PATCH 528/540] Query: cache QueryResult so each call gets the same one. --- rd_ui/app/scripts/services/resources.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index b97e7c570..49d6556fb 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -357,7 +357,10 @@ var queryResult = null; if (this.latest_query_data && ttl != 0) { - queryResult = new QueryResult({'query_result': this.latest_query_data}); + if (!this.queryResult) { + this.queryResult = new QueryResult({'query_result': this.latest_query_data}); + } + queryResult = this.queryResult; } else if (this.latest_query_data_id && ttl != 0) { queryResult = QueryResult.getById(this.latest_query_data_id); } else if (this.data_source_id) { From a40da45b1e516b9436bc68213a34776805d94a6f Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:35:07 +0300 Subject: [PATCH 529/540] Show filters in dashboards (if available). --- rd_ui/app/views/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/views/dashboard.html b/rd_ui/app/views/dashboard.html index b50b6559f..a2c0f7c08 100644 --- a/rd_ui/app/views/dashboard.html +++ b/rd_ui/app/views/dashboard.html @@ -14,6 +14,7 @@ +
        From 7b4c04024c2fb29cda289a3720917d2e7930f5ce Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:36:12 +0300 Subject: [PATCH 530/540] Use new getQuery accessor. --- rd_ui/app/scripts/controllers/dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index b1daba4dd..728ad89d8 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -77,7 +77,7 @@ Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id); Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id); - $scope.query = new Query($scope.widget.visualization.query); + $scope.query = $scope.widget.getQuery(); $scope.queryResult = $scope.query.getQueryResult(); $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow(); From bcce69904d11838f2530c8b24d4c102dcb5113fd Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:40:50 +0300 Subject: [PATCH 531/540] Global filters flag for dashboard. --- migrations/add_global_filters_to_dashboard.py | 12 ++++++++++++ redash/models.py | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 migrations/add_global_filters_to_dashboard.py diff --git a/migrations/add_global_filters_to_dashboard.py b/migrations/add_global_filters_to_dashboard.py new file mode 100644 index 000000000..3b78f6270 --- /dev/null +++ b/migrations/add_global_filters_to_dashboard.py @@ -0,0 +1,12 @@ +from playhouse.migrate import Migrator +from redash import db +from redash import models + + +if __name__ == '__main__': + db.connect_db() + migrator = Migrator(db.database) + with db.database.transaction(): + migrator.add_column(models.Dashboard, models.Dashboard.dashboard_filters_enabled, 'dashboard_filters_enabled') + + db.close_db(None) \ No newline at end of file diff --git a/redash/models.py b/redash/models.py index bdc061e07..70f2cecb6 100644 --- a/redash/models.py +++ b/redash/models.py @@ -248,6 +248,7 @@ class Dashboard(BaseModel): user_email = peewee.CharField(max_length=360, null=True) user = peewee.ForeignKeyField(User) layout = peewee.TextField() + dashboard_filters_enabled = peewee.BooleanField(default=False) is_archived = peewee.BooleanField(default=False, index=True) created_at = peewee.DateTimeField(default=datetime.datetime.now) @@ -292,6 +293,7 @@ class Dashboard(BaseModel): 'name': self.name, 'user_id': self._data['user'], 'layout': layout, + 'dashboard_filters_enabled': self.dashboard_filters_enabled, 'widgets': widgets_layout } From 77d628d2db60eee5a851dd4730059625976dfd3c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 18:46:38 +0300 Subject: [PATCH 532/540] Support for dashboard filters in the UI. --- rd_ui/app/scripts/controllers/dashboard.js | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/scripts/controllers/dashboard.js b/rd_ui/app/scripts/controllers/dashboard.js index 728ad89d8..30a9c133f 100644 --- a/rd_ui/app/scripts/controllers/dashboard.js +++ b/rd_ui/app/scripts/controllers/dashboard.js @@ -6,11 +6,40 @@ $scope.refreshRate = 60; $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) { $scope.$parent.pageTitle = dashboard.name; + var filters = {}; + $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) { return _.map(row, function (widget) { - return new Widget(widget); + var w = new Widget(widget); + + if (w.visualization && dashboard.dashboard_filters_enabled) { + var queryFilters = w.getQuery().getQueryResult().getFilters(); + _.each(queryFilters, function (filter) { + if (!_.has(filters, filter.name)) { + // TODO: first object should be a copy, otherwise one of the chart filters behaves different than the others. + filters[filter.name] = filter; + filters[filter.name].originFilters = []; + + $scope.$watch(function() { return filter.current }, function (value) { + _.each(filter.originFilters, function(originFilter) { + originFilter.current = value; + }) + }); + + }; + + // TODO: merge values. + filters[filter.name].originFilters.push(filter); + }); + } + + return w; }); }); + + if (dashboard.dashboard_filters_enabled) { + $scope.filters = _.values(filters); + } }); var autoRefresh = function() { From e7331633a443612073ccd31693bb0a0be5754626 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 19:32:17 +0300 Subject: [PATCH 533/540] Fix indentation. --- rd_ui/app/scripts/visualizations/table.js | 172 +++++++++++----------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index 70dfd0005..af3ec5362 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -1,90 +1,90 @@ (function () { - var tableVisualization = angular.module('redash.visualization'); + var tableVisualization = angular.module('redash.visualization'); - tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) { - VisualizationProvider.registerVisualization({ - type: 'TABLE', - name: 'Table', - renderTemplate: '', - skipTypes: true + tableVisualization.config(['VisualizationProvider', function (VisualizationProvider) { + VisualizationProvider.registerVisualization({ + type: 'TABLE', + name: 'Table', + renderTemplate: '', + skipTypes: true + }); + }]); + + tableVisualization.directive('gridRenderer', function () { + return { + restrict: 'E', + scope: { + queryResult: '=', + itemsPerPage: '=' + }, + templateUrl: "/views/grid_renderer.html", + replace: false, + controller: ['$scope', function ($scope) { + $scope.gridColumns = []; + $scope.gridData = []; + $scope.gridConfig = { + isPaginationEnabled: true, + itemsByPage: $scope.itemsPerPage || 15, + maxSize: 8 + }; + + $scope.$watch('queryResult && queryResult.getData()', function (data) { + if (!data) { + return; + } + + if ($scope.queryResult.getData() == null) { + $scope.gridColumns = []; + $scope.gridData = []; + $scope.filters = []; + } else { + $scope.filters = $scope.queryResult.getFilters(); + + var prepareGridData = function (data) { + var gridData = _.map(data, function (row) { + var newRow = {}; + _.each(row, function (val, key) { + newRow[$scope.queryResult.getColumnCleanName(key)] = val; + }) + return newRow; + }); + + return gridData; + }; + + $scope.gridData = prepareGridData($scope.queryResult.getData()); + + $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { + var columnDefinition = { + 'label': $scope.queryResult.getColumnFriendlyNames()[i], + 'map': col + }; + + var rawData = $scope.queryResult.getRawData(); + + if (rawData.length > 0) { + var exampleData = rawData[0][col]; + if (angular.isNumber(exampleData)) { + columnDefinition['formatFunction'] = 'number'; + columnDefinition['formatParameter'] = 2; + } else if (moment.isMoment(exampleData)) { + columnDefinition['formatFunction'] = function (value) { + // TODO: this is very hackish way to determine if we need + // to show the value as a time or date only. Better solution + // is to complete #70 and use the information it returns. + if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) { + return value.format("DD/MM/YY HH:mm"); + } + return value.format("DD/MM/YY"); + } + } + } + + return columnDefinition; + }); + } }); - }]); - - tableVisualization.directive('gridRenderer', function () { - return { - restrict: 'E', - scope: { - queryResult: '=', - itemsPerPage: '=' - }, - templateUrl: "/views/grid_renderer.html", - replace: false, - controller: ['$scope', function ($scope) { - $scope.gridColumns = []; - $scope.gridData = []; - $scope.gridConfig = { - isPaginationEnabled: true, - itemsByPage: $scope.itemsPerPage || 15, - maxSize: 8 - }; - - $scope.$watch('queryResult && queryResult.getData()', function (data) { - if (!data) { - return; - } - - if ($scope.queryResult.getData() == null) { - $scope.gridColumns = []; - $scope.gridData = []; - $scope.filters = []; - } else { - $scope.filters = $scope.queryResult.getFilters(); - - var prepareGridData = function(data) { - var gridData = _.map(data, function (row) { - var newRow = {}; - _.each(row, function (val, key) { - newRow[$scope.queryResult.getColumnCleanName(key)] = val; - }) - return newRow; - }); - - return gridData; - }; - - $scope.gridData = prepareGridData($scope.queryResult.getData()); - - $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { - var columnDefinition = { - 'label': $scope.queryResult.getColumnFriendlyNames()[i], - 'map': col - }; - - var rawData = $scope.queryResult.getRawData(); - - if (rawData.length > 0) { - var exampleData = rawData[0][col]; - if (angular.isNumber(exampleData)) { - columnDefinition['formatFunction'] = 'number'; - columnDefinition['formatParameter'] = 2; - } else if (moment.isMoment(exampleData)) { - columnDefinition['formatFunction'] = function(value) { - // TODO: this is very hackish way to determine if we need - // to show the value as a time or date only. Better solution - // is to complete #70 and use the information it returns. - if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) { - return value.format("DD/MM/YY HH:mm"); - } - return value.format("DD/MM/YY"); - } - } - } - - return columnDefinition; - }); - } - }); - }] - } - }) + }] + } + }) }()); \ No newline at end of file From 0ffda9d00266431415d4f56d0009230ecf253f4b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 19:44:28 +0300 Subject: [PATCH 534/540] Populate the column type field. --- redash/data/query_runner_pg.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/redash/data/query_runner_pg.py b/redash/data/query_runner_pg.py index cf094e461..8830b1bc5 100644 --- a/redash/data/query_runner_pg.py +++ b/redash/data/query_runner_pg.py @@ -13,6 +13,24 @@ import psycopg2 from redash.utils import JSONEncoder +types_map = { + 20: 'integer', + 21: 'integer', + 23: 'integer', + 700: 'float', + 1700: 'float', + 701: 'float', + 16: 'boolean', + 1082: 'date', + 1114: 'datetime', + 1184: 'datetime', + 1014: 'string', + 1015: 'string', + 1008: 'string', + 1009: 'string', + 2951: 'string' +} + def pg(connection_string): def column_friendly_name(column_name): @@ -61,7 +79,7 @@ def pg(connection_string): columns.append({ 'name': column_name, 'friendly_name': column_friendly_name(column_name), - 'type': None + 'type': types_map.get(column.type_code, None) }) rows = [dict(zip(column_names, row)) for row in cursor] From 52f44588e6f44b10a8a6bf0315d7c43e5f44ca1c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 19:44:52 +0300 Subject: [PATCH 535/540] Use column type (if available) to better render tables. --- rd_ui/app/scripts/services/resources.js | 19 ++++++---- rd_ui/app/scripts/visualizations/table.js | 44 +++++++++++++++-------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index 49d6556fb..910640003 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -197,15 +197,22 @@ QueryResult.prototype.getColumns = function () { if (this.columns == undefined && this.query_result.data) { - this.columns = _.map(this.query_result.data.columns, function (v) { + this.columns = this.query_result.data.columns; + } + + return this.columns; + } + + QueryResult.prototype.getColumnNames = function () { + if (this.columnNames == undefined && this.query_result.data) { + this.columnNames = _.map(this.query_result.data.columns, function (v) { return v.name; }); } - return this.columns; + return this.columnNames; } - QueryResult.prototype.getColumnNameWithoutType = function (column) { var parts = column.split('::'); return parts[0]; @@ -239,13 +246,13 @@ } QueryResult.prototype.getColumnCleanNames = function () { - return _.map(this.getColumns(), function (col) { + return _.map(this.getColumnNames(), function (col) { return this.getColumnCleanName(col); }, this); } QueryResult.prototype.getColumnFriendlyNames = function () { - return _.map(this.getColumns(), function (col) { + return _.map(this.getColumnNames(), function (col) { return this.getColumnFriendlyName(col); }, this); } @@ -261,7 +268,7 @@ QueryResult.prototype.prepareFilters = function () { var filters = []; var filterTypes = ['filter', 'multi-filter']; - _.each(this.getColumns(), function (col) { + _.each(this.getColumnNames(), function (col) { var type = col.split('::')[1] if (_.contains(filterTypes, type)) { // filter found diff --git a/rd_ui/app/scripts/visualizations/table.js b/rd_ui/app/scripts/visualizations/table.js index af3ec5362..962c10226 100644 --- a/rd_ui/app/scripts/visualizations/table.js +++ b/rd_ui/app/scripts/visualizations/table.js @@ -54,32 +54,48 @@ $scope.gridData = prepareGridData($scope.queryResult.getData()); + var columns = $scope.queryResult.getColumns(); $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) { var columnDefinition = { 'label': $scope.queryResult.getColumnFriendlyNames()[i], 'map': col }; - var rawData = $scope.queryResult.getRawData(); + var columnType = columns[i].type; - if (rawData.length > 0) { - var exampleData = rawData[0][col]; - if (angular.isNumber(exampleData)) { - columnDefinition['formatFunction'] = 'number'; - columnDefinition['formatParameter'] = 2; - } else if (moment.isMoment(exampleData)) { - columnDefinition['formatFunction'] = function (value) { - // TODO: this is very hackish way to determine if we need - // to show the value as a time or date only. Better solution - // is to complete #70 and use the information it returns. - if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) { - return value.format("DD/MM/YY HH:mm"); + if (!columnType) { + var rawData = $scope.queryResult.getRawData(); + + if (rawData.length > 0) { + var exampleData = rawData[0][col]; + if (angular.isNumber(exampleData)) { + columnType = 'float'; + } else if (moment.isMoment(exampleData)) { + if (exampleData._i.match(/^\d{4}-\d{2}-\d{2}T/)) { + columnType = 'datetime'; + } else { + columnType = 'date'; } - return value.format("DD/MM/YY"); } } } + if (columnType === 'integer') { + columnDefinition.formatFunction = 'number'; + columnDefinition.formatParameter = 0; + } else if (columnType === 'float') { + columnDefinition.formatFunction = 'number'; + columnDefinition.formatParameter = 2; + } else if (columnType === 'date') { + columnDefinition.formatFunction = function (value) { + return value.format("DD/MM/YY"); + }; + } else if (columnType === 'datetime') { + columnDefinition.formatFunction = function (value) { + return value.format("DD/MM/YY HH:mm"); + }; + } + return columnDefinition; }); } From 38293fc1550f9c526b0385c4830fe5efef6d08c1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 5 May 2014 19:45:07 +0300 Subject: [PATCH 536/540] Fix: query save fails if query has queryResult property. --- rd_ui/app/scripts/controllers/query_view.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index b957905ee..f753fd9a6 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -33,6 +33,7 @@ }, options); delete $scope.query.latest_query_data; + delete $scope.query.queryResult; return Query.save(data, function() { growl.addSuccessMessage(options.successMessage); From 4d4f41733deb8aef6a016f68334823731559306a Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 7 May 2014 15:13:29 +0300 Subject: [PATCH 537/540] Report to statsd request render time --- redash/controllers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/redash/controllers.py b/redash/controllers.py index e9195235a..b489c10c2 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -19,7 +19,7 @@ from flask_login import current_user, login_user, logout_user import sqlparse import events from permissions import require_permission -from redash import settings, utils, __version__ +from redash import settings, utils, __version__, statsd_client from redash import data from redash import app, auth, api, redis_connection, data_manager @@ -129,6 +129,11 @@ class BaseResource(Resource): def current_user(self): return current_user._get_current_object() + def dispatch_request(self, *args, **kwargs): + with statsd_client.timer('requests.{}.{}'.format(request.endpoint, request.method.lower())): + response = super(BaseResource, self).dispatch_request(*args, **kwargs) + return response + class EventAPI(BaseResource): def post(self): From 05c915cf00fc1e2a80041988c348ad51551783a5 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 7 May 2014 15:37:08 +0300 Subject: [PATCH 538/540] Fix indendentation --- rd_ui/app/scripts/app.js | 112 +++---- rd_ui/app/scripts/controllers/controllers.js | 291 ++++++++++--------- 2 files changed, 204 insertions(+), 199 deletions(-) diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index ac8a4b11d..c87d61896 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -15,65 +15,65 @@ angular.module('redash', [ 'smartTable.table', 'ngResource', 'ngRoute' -]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', - function($routeProvider, $locationProvider, $compileProvider, growlProvider) { + ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider', + function ($routeProvider, $locationProvider, $compileProvider, growlProvider) { - function getQuery(Query, $route) { - var query = Query.get({'id': $route.current.params.queryId }); - return query.$promise; - }; + function getQuery(Query, $route) { + var query = Query.get({'id': $route.current.params.queryId }); + return query.$promise; + }; - $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/); - $locationProvider.html5Mode(true); - growlProvider.globalTimeToLive(2000); + $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/); + $locationProvider.html5Mode(true); + growlProvider.globalTimeToLive(2000); - $routeProvider.when('/dashboard/:dashboardSlug', { - templateUrl: '/views/dashboard.html', - controller: 'DashboardCtrl' - }); - $routeProvider.when('/queries', { - templateUrl: '/views/queries.html', - controller: 'QueriesCtrl', - reloadOnSearch: false - }); - $routeProvider.when('/queries/new', { - templateUrl: '/views/query.html', - controller: 'QuerySourceCtrl', - reloadOnSearch: false, - resolve: { - 'query': ['Query', function newQuery(Query) { - return Query.newQuery(); - }] - } - }); - $routeProvider.when('/queries/:queryId', { - templateUrl: '/views/query.html', - controller: 'QueryViewCtrl', - reloadOnSearch: false, - resolve: { - 'query': ['Query', '$route', getQuery] - } - }); - $routeProvider.when('/queries/:queryId/source', { - templateUrl: '/views/query.html', - controller: 'QuerySourceCtrl', - reloadOnSearch: false, - resolve: { - 'query': ['Query', '$route', getQuery] - } - }); - $routeProvider.when('/admin/status', { - templateUrl: '/views/admin_status.html', - controller: 'AdminStatusCtrl' - }); - $routeProvider.when('/', { - templateUrl: '/views/index.html', - controller: 'IndexCtrl' - }); - $routeProvider.otherwise({ - redirectTo: '/' - }); + $routeProvider.when('/dashboard/:dashboardSlug', { + templateUrl: '/views/dashboard.html', + controller: 'DashboardCtrl' + }); + $routeProvider.when('/queries', { + templateUrl: '/views/queries.html', + controller: 'QueriesCtrl', + reloadOnSearch: false + }); + $routeProvider.when('/queries/new', { + templateUrl: '/views/query.html', + controller: 'QuerySourceCtrl', + reloadOnSearch: false, + resolve: { + 'query': ['Query', function newQuery(Query) { + return Query.newQuery(); + }] + } + }); + $routeProvider.when('/queries/:queryId', { + templateUrl: '/views/query.html', + controller: 'QueryViewCtrl', + reloadOnSearch: false, + resolve: { + 'query': ['Query', '$route', getQuery] + } + }); + $routeProvider.when('/queries/:queryId/source', { + templateUrl: '/views/query.html', + controller: 'QuerySourceCtrl', + reloadOnSearch: false, + resolve: { + 'query': ['Query', '$route', getQuery] + } + }); + $routeProvider.when('/admin/status', { + templateUrl: '/views/admin_status.html', + controller: 'AdminStatusCtrl' + }); + $routeProvider.when('/', { + templateUrl: '/views/index.html', + controller: 'IndexCtrl' + }); + $routeProvider.otherwise({ + redirectTo: '/' + }); } -]); \ No newline at end of file + ]); \ No newline at end of file diff --git a/rd_ui/app/scripts/controllers/controllers.js b/rd_ui/app/scripts/controllers/controllers.js index dcb4672b7..aed2dede3 100644 --- a/rd_ui/app/scripts/controllers/controllers.js +++ b/rd_ui/app/scripts/controllers/controllers.js @@ -1,157 +1,162 @@ (function () { - var QueriesCtrl = function($scope, $http, $location, $filter, Query) { - $scope.$parent.pageTitle = "All Queries"; - $scope.gridConfig = { - isPaginationEnabled: true, - itemsByPage: 50, - maxSize: 8, - isGlobalSearchActivated: true - } - - $scope.allQueries = []; - $scope.queries = []; - - var dateFormatter = function (value) { - if (!value) return "-"; - return value.format("DD/MM/YY HH:mm"); - } - - var filterQueries = function() { - $scope.queries = _.filter($scope.allQueries, function(query) { - if (!$scope.selectedTab) { - return false; - } - - if ($scope.selectedTab.key == 'my') { - return query.user.id == currentUser.id && query.name != 'New Query'; - } else if ($scope.selectedTab.key == 'drafts') { - return query.user.id == currentUser.id && query.name == 'New Query'; - } - - return query.name != 'New Query'; - }); - } - - Query.query(function(queries) { - $scope.allQueries = _.map(queries, function(query) { - query.created_at = moment(query.created_at); - query.last_retrieved_at = moment(query.last_retrieved_at); - return query; - }); - - filterQueries(); - }); - - $scope.gridColumns = [ - { - "label": "Name", - "map": "name", - "cellTemplateUrl": "/views/queries_query_name_cell.html" - }, - { - 'label': 'Created By', - 'map': 'user.name' - }, - { - 'label': 'Created At', - 'map': 'created_at', - 'formatFunction': dateFormatter - }, - { - 'label': 'Runtime (avg)', - 'map': 'avg_runtime', - 'formatFunction': function(value) { - return $filter('durationHumanize')(value); - } - }, - { - 'label': 'Runtime (min)', - 'map': 'min_runtime', - 'formatFunction': function(value) { - return $filter('durationHumanize')(value); - } - }, - { - 'label': 'Runtime (max)', - 'map': 'max_runtime', - 'formatFunction': function(value) { - return $filter('durationHumanize')(value); - } - }, - { - 'label': 'Last Executed At', - 'map': 'last_retrieved_at', - 'formatFunction': dateFormatter - }, - { - 'label': 'Times Executed', - 'map': 'times_retrieved' - }, - { - 'label': 'Update Schedule', - 'map': 'ttl', - 'formatFunction': function(value) { - return $filter('refreshRateHumanize')(value); - } - } - ] - $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}]; - - $scope.$watch('selectedTab', function(tab) { - if (tab) { - $scope.$parent.pageTitle = tab.name; - } - - filterQueries(); - }); + var QueriesCtrl = function ($scope, $http, $location, $filter, Query) { + $scope.$parent.pageTitle = "All Queries"; + $scope.gridConfig = { + isPaginationEnabled: true, + itemsByPage: 50, + maxSize: 8, + isGlobalSearchActivated: true } - var MainCtrl = function ($scope, Dashboard, notifications) { - $scope.dashboards = []; - $scope.reloadDashboards = function() { - Dashboard.query(function (dashboards) { - $scope.dashboards = _.sortBy(dashboards, "name"); - $scope.allDashboards = _.groupBy($scope.dashboards, function(d) { - parts = d.name.split(":"); - if (parts.length == 1) { - return "Other"; - } - return parts[0]; - }); - $scope.otherDashboards = $scope.allDashboards['Other'] || []; - $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other'); - }); + $scope.allQueries = []; + $scope.queries = []; + + var dateFormatter = function (value) { + if (!value) return "-"; + return value.format("DD/MM/YY HH:mm"); + } + + var filterQueries = function () { + $scope.queries = _.filter($scope.allQueries, function (query) { + if (!$scope.selectedTab) { + return false; } - $scope.reloadDashboards(); - - $scope.currentUser = currentUser; - $scope.newDashboard = { - 'name': null, - 'layout': null + if ($scope.selectedTab.key == 'my') { + return query.user.id == currentUser.id && query.name != 'New Query'; + } else if ($scope.selectedTab.key == 'drafts') { + return query.user.id == currentUser.id && query.name == 'New Query'; } - $(window).click(function () { - notifications.getPermissions(); + return query.name != 'New Query'; + }); + } + + Query.query(function (queries) { + $scope.allQueries = _.map(queries, function (query) { + query.created_at = moment(query.created_at); + query.last_retrieved_at = moment(query.last_retrieved_at); + return query; + }); + + filterQueries(); + }); + + $scope.gridColumns = [ + { + "label": "Name", + "map": "name", + "cellTemplateUrl": "/views/queries_query_name_cell.html" + }, + { + 'label': 'Created By', + 'map': 'user.name' + }, + { + 'label': 'Created At', + 'map': 'created_at', + 'formatFunction': dateFormatter + }, + { + 'label': 'Runtime (avg)', + 'map': 'avg_runtime', + 'formatFunction': function (value) { + return $filter('durationHumanize')(value); + } + }, + { + 'label': 'Runtime (min)', + 'map': 'min_runtime', + 'formatFunction': function (value) { + return $filter('durationHumanize')(value); + } + }, + { + 'label': 'Runtime (max)', + 'map': 'max_runtime', + 'formatFunction': function (value) { + return $filter('durationHumanize')(value); + } + }, + { + 'label': 'Last Executed At', + 'map': 'last_retrieved_at', + 'formatFunction': dateFormatter + }, + { + 'label': 'Times Executed', + 'map': 'times_retrieved' + }, + { + 'label': 'Update Schedule', + 'map': 'ttl', + 'formatFunction': function (value) { + return $filter('refreshRateHumanize')(value); + } + } + ] + $scope.tabs = [ + {"name": "My Queries", "key": "my"}, + {"key": "all", "name": "All Queries"}, + {"key": "drafts", "name": "Drafts"} + ]; + + $scope.$watch('selectedTab', function (tab) { + if (tab) { + $scope.$parent.pageTitle = tab.name; + } + + filterQueries(); + }); + } + + var MainCtrl = function ($scope, Dashboard, notifications) { + + $scope.dashboards = []; + $scope.reloadDashboards = function () { + Dashboard.query(function (dashboards) { + $scope.dashboards = _.sortBy(dashboards, "name"); + $scope.allDashboards = _.groupBy($scope.dashboards, function (d) { + parts = d.name.split(":"); + if (parts.length == 1) { + return "Other"; + } + return parts[0]; }); + $scope.otherDashboards = $scope.allDashboards['Other'] || []; + $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other'); + }); } - var IndexCtrl = function($scope, Events, Dashboard) { - Events.record(currentUser, "view", "page", "homepage"); - $scope.$parent.pageTitle = "Home"; + $scope.reloadDashboards(); - $scope.archiveDashboard = function(dashboard) { - if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) { - Events.record(currentUser, "archive", "dashboard", dashboard.id); - dashboard.$delete(function() { - $scope.$parent.reloadDashboards(); - }); - } - } + $scope.currentUser = currentUser; + $scope.newDashboard = { + 'name': null, + 'layout': null } - angular.module('redash.controllers', []) - .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) - .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl]) - .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); + $(window).click(function () { + notifications.getPermissions(); + }); + } + + var IndexCtrl = function ($scope, Events, Dashboard) { + Events.record(currentUser, "view", "page", "homepage"); + $scope.$parent.pageTitle = "Home"; + + $scope.archiveDashboard = function (dashboard) { + if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) { + Events.record(currentUser, "archive", "dashboard", dashboard.id); + dashboard.$delete(function () { + $scope.$parent.reloadDashboards(); + }); + } + } + } + + angular.module('redash.controllers', []) + .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl]) + .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl]) + .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]); })(); From 509edf651b74d416df207e694a8412d5ee1cd60d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 7 May 2014 17:25:43 +0300 Subject: [PATCH 539/540] Add bucky (client side metrics client). --- rd_ui/app/index.html | 3 +++ rd_ui/app/scripts/app.js | 8 ++++++++ rd_ui/app/scripts/controllers/controllers.js | 7 +++++++ rd_ui/bower.json | 3 ++- redash/controllers.py | 16 ++++++++++++++++ redash/settings.py | 1 + 6 files changed, 37 insertions(+), 1 deletion(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 8f5cdcbeb..8fc2f8785 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -114,6 +114,7 @@ + @@ -139,6 +140,8 @@