From d455409cc929135cf571646f978a314d13d0f38c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 7 Oct 2021 11:15:34 -0700 Subject: [PATCH] issue-696: Added trino support for Openmetadata (#697) * issue-696: Added trino support for Openmetadata * issue-696: fixed linting issues * issue-696: not mentioning Trino for now as it will be part of 0.5 release Co-authored-by: jbuoncri --- .../entity/services/databaseService.json | 4 + .../ui/src/assets/img/service-icon-trino.png | Bin 0 -> 21402 bytes .../ui/src/constants/services.const.ts | 3 + .../resources/ui/src/enums/service.enum.ts | 1 + .../api/services/createDatabaseService.ts | 1 + .../entity/services/databaseService.ts | 1 + .../resources/ui/src/utils/ServiceUtils.ts | 5 + docs/SUMMARY.md | 1 + .../connectors/database-services/trino.md | 95 ++++++++++++++++++ .../schemas/entities/databaseservice.md | 3 +- docs/roadmap.md | 1 + .../sample_data/pipelines/pipelines.json | 7 ++ .../examples/sample_data/pipelines/tasks.json | 8 ++ ingestion/examples/workflows/trino.json | 29 ++++++ ingestion/setup.py | 1 + .../schema/entity/services/databaseService.py | 1 + .../src/metadata/ingestion/source/superset.py | 2 + .../src/metadata/ingestion/source/trino.py | 48 +++++++++ roadmap.md | 1 + 19 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-trino.png create mode 100644 docs/install/metadata-ingestion/connectors/database-services/trino.md create mode 100644 ingestion/examples/workflows/trino.json create mode 100644 ingestion/src/metadata/ingestion/source/trino.py diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json index dc87653e184..3af01dcaae8 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json @@ -19,6 +19,7 @@ "Oracle", "Athena", "Presto", + "Trino", "Vertica" ], "javaEnums": [ @@ -52,6 +53,9 @@ { "name": "Presto" }, + { + "name": "Trino" + }, { "name": "Vertica" } diff --git a/catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-trino.png b/catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-trino.png new file mode 100644 index 0000000000000000000000000000000000000000..ffbd80474066d0ee201ed37878d4d2043f50f4ab GIT binary patch literal 21402 zcmXtAby!s0*BwF{97+KJVJJ!I?ve&+hI9aFk(82-p&O);ZX94pDH&1$Y3YzgK#QT2?^iP6F~{%(GxYzL({GEes_U(T|I+y_YcD- zC+Gh}0=V5h403!sJPsw!IV}5PwEl|UGs*iAp!pC7f&+#7%KB`JT3bac%FF4^N*Rvs zDGDOc#NQ#Ion@!(=~~~uVD`Hh@S&by($VxM4!}i zyN8F*Ny_Cv^;*++xmM@ZhLYeBL9Pho#KS<7Q&Xv=+QfthEhjz_^wNf<6SszYTO6N5 zgM)cIpQCHJ>FA6oF%`768D}GxeqM>FzW8iWhIS!{esUyA6DIpPn=~2@*n%= zbEuRvE)y9hyqOrStgw*bWf3GQ3kfzKyYg0jL4PQXx|a|9f@saelo@!2z?7kr7$rqN zNiE=2dEvtyh{TazTwL^KNX5yTQo)fHMX6NhwS|%#W94pj_JT2{u;k2a*oiSPFksCS z@2d0KAO3{(mkwRa*VlJRnduw%w$HgBKfj;Z-qy>{q1hquk+*P8>i&&k(ie|Z9rjjS z!7D=0z5GxTS0dUYC}w_IT3T>o6uhjd)aJbiyjk9XCPhnjBnV!+DLN!e_-*xzQk7=I zvJ1znoq7wz9J`&)iQ)BHylr=Hk6jo6Yk|h7k3pX@td??yPajcM_A8+|Y zax`2$KIp`MSIEi9v8sm-TJiRhR=!)Zj;i96Tcp;3qy1wqsUwHcP z?Us-!`&&rF#TQ|*9pShteRl8mr3S&(^Xt(nD-c}HL{Z+x?pCd$``^uD;y|ir&jxqg zwdueEQprhRyfFDk2Q$#;OzT)Nh-HX0DLS}z;&oP!OrjE_<@{eS@iikBp1>)7?SWp3 ztT@o**ph}|mbwRLv`NU!4xBy{IsFP(POc$pCUe^?l4Iip^%CS0p$!;BL|vIXK=5pGzcb}p)33QbEW~=-z4eeT5Dl%Y*O57E_z9_ijjmh}9S({| zkzlM0Ny{iBZ^O_PC{xQ~;NEw9>6am~E)5rbCnck9DaZu>-FnOD#W&w@b?bkmu3Lm@`fphwP!TcdC0_2T=gUh|QCW$LwgBCeB@xXz=v3$ro7iE@8crK)HKf2vyxnbQ zEg~s)>tms0t#nrszaeJQBb8&UoXV80ki~MIQIyMVn2uniuq4Nkb{a#vltlFG3(=%9 z=}ml6pLCyQ#-o_RKEVs7BiSpOZe^t>UsGdJsF0m}75U24Y5xoJG>qr3zHS2aq;tAR zjm>4cb?R}sh=~!1kLuyJDG)sy8ylJ3LR29$RoG{w8DkLqUK+H!vqPUX zTypi^@NaneshF59@%J0#K4a)4y9AVI_%pCpVT`_CLJF_K1ne}YmX>mp%c9hXk;#J9 z9t^nQ1AM4oF~w)^o6kiG*|a0q#A-UY)!lVO^MUs$Fh~EH!{DXG)Z{I36N2bZBZbWL~w_FND$xz18?igRR*616Wd?v>nWmKC5FoNGwp<(RbElreqrsCL=dTn zmmhnBvWeD2oRlv(Q1TuPZg^YP24`y0#_UyDTo|>5=Mh6q;6e8mk z`QY0eh#y`$Iy&g(bs}klOs&0MGJOXVP)$~gm|HwF62Ss{-UynR=L4S%IhVamk>m0( z7wQkW)1p+vE8sk75U{1H^m>glm@UDET=6_-Zq=tL!M0MXzj%MXtuG#?CK79(bk~1d zM8fn(>jRZJB<&3R?_bQ+6yW0mVwFiDZ@6)0q&V=u@&@+zMm5=&o_8QmjS?&DelQ=? zx|V4}bu0@FAFujC_iXtgP~8_*TbG7^@?L~-MYBB08)d;-q5TOpNLm34F^(GUXdlnt z`3XC~POB>oiiHymsrx^gJJ7@mZliHY2b#qpFx$Y3yJ3=*NwQTD0+#j2>2yZ0g%mpD z3(Y|fHcZImO<1)N>GxA1kKAq&L_^S^)s+p;-AnHh;6!JpIw*xxxbQgchoI^1c}vO! zsW>w(zuGrSewga+CwG(7HB^OokG_in+!irisBNR?*=Cx&CaVn&X^8OXwZh!4i2rx- zYu~dsV%Nx?f&@X`DVDF|4fFAjGz`PQFiiJ5DVYAOrvyCV5+ESwkfU7i`Sa>O9O=!? z%}^b8riR<2p9yg5wm&p_L+qT7@bQT(n9;Rj$3E3zQWMd3d2OHiz~E?wV;7pxJ#`Ww zq0@QKX+9lEv1?GMYw*8U2i_vCeE$>8Bm^X5RvKMQ*o!^AiT*jw-?h^UvejH@bw4G! z`{|hYk$c9gZ|7%cQT0=v)u+t(K$24NG9qy)Z?5{qq+dRtGC$djUTE&+c26v)eC0))3OfI{#P;o7wY&N>VkoyVDM z#_wzS%kRwG-0CmtXljuMWZfWID>yEmUKHt{Pgk!s;N*41=iI5XnaB7%DZ_@QIj%U_ z)%B3^eHNQbY|7c-IVI}OWle%aaDHOAuh?@UmyEdIy4d=nHH&$qut$3n>H5#6S!Woi zV-Tv~u6~|0@DQ_TZCzv)&ilnlpSvu|*`P~@d1ai6`JUcW^{f))-8ee2xw<5^J`}y- zj*hQ)P&rWbkPtwJtitf(OcLYbn z_8~HQ{;^<^vYl+*QH>6`HT4g=GyCd77&>uIFK8(#s_M(EW5&}NGPp^=@46-oGgIWo zCZBI*Ido5p9DD?zDp8ph*Ej6+ZUGwqZKt=ClDQ7U9Co>Z0}VMT_y`R%ABZ?9aw}?k zAiN;B6^$}-{OmfU!k?vP&Pjssygd~M(}FpfRi@jOOaGpQ<`L7A+6iLxhKD)GoPdRY z?76dESEx-$hX#Q~-VKP437FlKPTiJlqsHm=?W^UcEa04j8J}Gp97m1Qf=N{ST~J_$ zhUwQl`9v$v2*$!eh_{bsY0+OT$hh8Y-qObgT!&g#_*XWSzRIqaEJ+-1clq(D+utyj zkPtU2O{dOR-Bc!0078WR3iJ@lz%debn#IiLTtT|+XmzG{2MK*0m(j+03dT6W(oN~>6>0>qljiAIp8=#({mu0ms+FR1ZSFEz&e!+IiS?$4}lvw#Ol^YD1Pdy~` zUZ{TBUc9@&2F8J>CS1FVE`Dg3wp0Y`D6Z|HSc_xa)l{CJr?SI(r}!wmoh#o2o9tJH z2K`Jp*99`_>xtQ{!TZQo*)Op+Yw{81M)mG+S~6`12M4uPjZ&SnH#wGk_no2Z>mD@R$+K-=l0Af00^zcoTg0sP(@0kT5zO2NVfDUeuEwr#Id$S5+ZNj3*mO05Yw!_|3na? z9P{l<6?qvJ5Ll&Jn%hYbDOctjnTA~HsxkP!(!dP(>DDxf>B@dLr!ZMClc%*lf73#F z=q8wauj4V8RL#k`hJX5~`t*~4a9kUIhM~%A!L;fU7X?C8OsqF^;e&m3X<{hge6@7a z$&(#y$v150>@8wYms``)1EF3$yn2BA5o zM`P$dt6P`Y&fgqeXJ(%CpEA>vC}T*3Q+!ht4Sd!x{X2n9frII3)rOSklNdw6N)shRwZWgVlFO2G`O8UpiFx(=$x?Jk8jg zBM39(e68>gbYF?gRYa$-d4M zvTm1!+GGbBvM9rj_|)Zmmo17dhw9Q*qWamJM~LsRWBM<5f^z** z4`#U~Pva3)D5w8hfO-TP+&GA^8t0ITPm0`^$0sHUc-DlJe(=VeCcS5BMXb4>^t z@^ZlQeu*`gZl_&qqqJR{wknns0FXh(`~7h%9nMC<((#p7v(WPUX)L2|hfvBFrZ2i?5mish>et;E*MfTLsrz~*Ssq?UOe zNnMd(f5pF=csIe^>F6a$cV!KZdZFP%ffGNf*~Da^8(WU-b~j zAn+**@@96l3)me6Q)6eBZ3kx!GdsV=1|xUZ9@7x@XT(?pj@nm$|0*42s4})ct?KrQ zqgbUGHs&kur|3XKCuA-F4!FB|hZ3Weh!K;`M>CrjO>2AJT(UwCH%ByB;v;CAz|o8SwTFAa@R5#gOvfF*DV zEU;Bh9z9_)UH1w6@cQ3YE z_u+Z;Mz@^!8WOtqWd-h&QHDV9bbHJ%a?;SvceaZNVYqkrxg9pk<~(<(&RYe^`*B<< z$q(hb`_s7n9JDJGnIcQ>|?sRzO^%%!GG9(?ounzP@P*KGf{l$t7ALGxV~ z{JQMZxO5#Fx|X`Sx{7F1%6AFJ=L2F0RGkfhMdEbp7|ED*>!_{{gP1XE_lL=hEJW*H zPCeyl7y%vn_A@G~jtb-*KQ?-EB_Y=C!GQ*aaJfo(uvB((cisJs#4Oq_*tDV0<0I`O z3@;hGM(sIcT>79YDSwB7-$jDWjQ~5unA&5(?WgImNQh4-&E8LOdUjT!lEfGAahu%l zPv;zqn389fI^%R+!*r~^Ma)N1G(Q2wLdJ4xqKLJnigOW54YR;pn$3nHfU0`xi}5%G z?;eGQMuDznZk|_>?6;>N7eLBJTzRVIQIa_Im^H!zq)Hq}&0C+Uy_ciZB*pTwvqq$J zJW0j9k`DzZI*NzxIT_0Qko9O=tl-s}_YZhK5VCo(??2m3lDrZFy01aLYmlDQnRocP z$E0lCG*3Pu>pEsX=s&g8Q0=+4xEq6*1xEOVMtmOxVpWw98MWYAdz2F(!Xah>a1!(& z0Ga|DwrGn+bo8d!jlGz$&!RmmQ%bA>Cpyhp@Aa>mKTa_pD&x1Q_AW6o41++E6BEh2 zaP3@~8B4i%5gmxWhlc=d8p|*{7 zYa(grT~joz^t$(*Ds<2B7eMIMo5aQIzTfJSoKL{kD$k7n^ZnF`gbSdq_Br3W4F#&S(}#E*d?jWl!;8fDAn(vl zMg<$U1IjtZ@|36>g0D`La~-?pcQUJulT*s#C|eKA_Y7;j&^?D=fP>sb|GLzy&~)O( z4EU5eGV~a616y8wO~LY_aoI&QCnM~o0%qNv6%G<@bWVQh_Ly)n66;##)t+9c2fT9v zYQTU+fLP9-DU~<>{Azs`pzLfLt%|>ew@>9Nj~QN70f7Yp|G``Bp{>N-_r%JLjsp`S zXWwSoe&R%ee!QgzP+OiiJac`FEa2#rW)m(w9=m+O>@J z%JkfyO(daUCQ>fD5UGLpsPC5tET>D2C^Squ6xw;N2n=#98w)8%#bv@TA_3yA004A} zE1tIs?ql>(&qUW05ms7$*4uyTA|M<|{Bz+07Tetft%{Tn{sZ4dGb+c?{hv0!yRp zaHKsmnx)GJo)&zbLTK%pZ=CvATIY+zT_dNjSn=RH4BF3Z%lT5$0xn;TL|P*av2+x{ z0iThQcxYJf(a{*Q`UYEmMbtIfw;d&L0C36TNA(Cro4Ugy86ia%K-?1H!jzrzBi3Bg z8T6xP#hd%rnpFX2v7?(p?D+%SV`w2raYLL}SC3wja4!Mx`f3C3l2cetY{0OJr>|&q z>NKAxGU&VU+s}diK8Nlx0L1F!q25PBEx*DN5>8uE$4Rs}c7{QoT7%WO#(&`k5D=0v zHR}nsC}lf)o#uTSLz>sSZ!Qbx4_^%eVPPi3cqUfwUHEu<(9j4`CorWlP*snBHAO%p zQ_z8k&34U0n?CaoZ+d5M<_@a`!8YjX`<#fnt$+^V5v4=npKugP#qh1 z;-=WC&Q!+4CaTLYM)7y2eRX=lBaDj_`U*UBLZr{JVioLV{wTd9+t!PuXxRCmcf%?( z?W3yCF@42zyS10#w0*yrfUPK_Pg_Wuvr(RM8*Lo27lpk}Z1R->VaTP#45viHnoRHB z_4zcKUglcG>Cu4!E@M_n_`-!yK)>H>oq)-_jt7IFzx9V;I6Z4m2_L z1N-$w)D$C#=j$A~6>(FYMc3^{Xm=Q$}Q^RTrSa^ezDRnN!l zs52vy-0pQM3XDUjDoPve=&yu#BcjKPDZMaiuKxVA!xUwCvdzMI7FiIy~ ztNVOm{SI}jm(gzhiWC752@gGjR!o?34(?T)4Y&iKmyh!#CmLkIp@v|-2w|DMEiM@} zf+O|v4GY(RH7(VENJHKg?>h))eo0A5=vzk~!1pg}nUYdF66xt3f77O>E^C z_xkBk<6sp!kR_b8Uf=aYcP`6m3q*emb9Edzy;(CmBx@MSIa1#OuQ1;1_4+E|3IAC?%L$uz& zNou0#qtv^kr|&~$s$N)W46$jvE2P?a3=kN*XE?4-eg&z9(cy+L+?@-@)fCxm^u$wH zkE!%I=R59IZ3Kf-`G+Z-Fp4}iq`M|t3Uqg2IruqWpl zNe$_QOv3(U_S8B$^^<)-nmX! z&b6-9p@#BRlFe%GMIm>&JdGgYCLVdSNN$3}6LHNcfF`df|9sUdYVT;9U((k{W>;2Q z+BydX_`=UXXTU2*BO?%`5$=UUoVN;Qp<9bbNpC`vSvT!9rwzU=Rad6xF?2cvojsjll5@-`)b}y%@G2v48`-kSyodlrPjl12>wr zHtJOuD$SUGjR1<;2J})for0%u%7t2Bawd84T`9dDUeNz*0cg{s(Md2&Yyk=oXbS@^ zJmu=@YAY&^jVIA>H6`0fuw<?bnQ({o3jSN5h{IAwQlSk1NGjaIx*#Gg?TL${ z6Z`@={^!JxAfni4)S_{!hD6NIJ4 zb3f1f^CQ!+z)il!-Y7O&H6=JMR_O)M1Y=q z+b=8Fm z-e5?a{jzl5%K4yeWJGED@pQ_4HSFp5VN)>nN(9Wq;a z0g|SEN4@U*$Qf;GYo_6Bk>p&TUDmmdk76`Z{w9q+`&uhfv&uPqn%bzGW)@N#cKI7S z$^K6~o>^WE1)&z)N{8*|v=SF9A@2W%g(iL>EbZnR>>7Md?SZar)QnI2lHqBYpX7Hq zI)-7QpJ6uj&Vi2_iz?gqc+cc>3;U1X99NFKwnH9E=99DQ|3pQS?j)&lB{HdYtbhGC zQkZvtv&C=OM*yvVOGpnJN!M(*Inx&%`qx4(Cmsa?j*!)1|Ay<>LW^5@Rh61H8$0_B zFqD>GN%)ly52SN4%i#x*cquH?BZG<|h$L(oHoBSPp2bOXYmN6bHc@wX{yuUYa7X78P?MF<*XN)6j?!y)vb>>) zWCPJpn|w>VoUyU8*WwpWOh|OJda;qOOOXBh>UP~HMHQ#_HGb)YviNA}%`ITUv24cK zf%fyItYePnA5UKEkD%p^ekl10@OwOs+y=yiq9|S-1tlWmM#tCcRYOB6vxm45J&(k; zX?ZMSaz?>nfWyryox@cNWRU2xpTRU@o-t3jOq8sxtxwjXUzYfle1Z!&&GA=MRTTjn zr{&?{q1Xzc5}z1<#Xv;?8Fg7G7NMAlaz{a%-sRZC^mKIxa%G}WvM5Wr_QL{nj0?Iu zHTL==+063V1=I?)Zd#ntKF&SsW# z8gp_wy$N)&6hL0#8Eu}LttJ{sRQ%?YjLEJNMv2-#=lEjz~U9~ z;&n=Qj}OtxnVME^wFN&TKX>h!LJCd!;qM57KVnzfIeFaFClN>Yv5-javJ}D`2;q-QT0g;`+kFQ~2FP z{)%e?lWf{WdPf?T*LfYm>>V6*ik;*8Nxz55q6~0X>h1e~S@GWTp%ZKiYv@J6mk(L$ zXVn0^u&z3}+VW)e`uftzk#a}^bsH~=D73r7$R7J5?5WwnK%fdr+td7Tp@olWTmi4! z;;&k}4C*6Lt0?aq@&$V1NS~gbR`iLGq97+|yFcFkd&~lK3b~-8}l??p) zl(|X1>W39@lLvz>-w)gf(8I*#l`%P2BSf4zm zunbTP%>TvDNdtJP)2}b|4u!15syQ*wrE=q_MGA)Uf14a*A=hCsACu{+c%Zo6Wp9iC zP0xcsgXbICLej$+^4dGXTu9p7nB&dSJSi*fRCsDe?&O~^m^vwV572|v3?1Dwqi4&5 z#{Y=}j&7apPW7BM7-U9dVe+KJP#n&ZO7+19pq&=|ulGfG(nt`z$(R4ORKru53Wrz0 zgbPKUo4i!9d6uB~SE}8KfJiIb$th~fF_=zEQ&aQD?Q^D+exPpZH~IuS3gkIs$NcLO zQx2I*WO!9pPyn$B*M{hC@_HXz0kxuvnEu+2FwiIE&ziYg0R!~!dTk?HO8OLMm$wi1 zqHFc7En*e{k&(4+crUWlw_6F)M&Vv zm~y34yhRm%SL7mAeMAk~f;Ig5=o%7Yl`ZNKHT3H0gc;`1gcPg zDyJKy|6*1R7{3c-_9U4+_2>Aw?C<{gm!Ig@GYxHWJf3{qGhQx;=4yr4w4Xt*_RQ~9 zl9|;DfMgMI(0*=~DdbEmakX9O4#cqWhcE`j^de*D_SCzRQmL^i19SEJ+vR(8^Sk5p zR~(3;p`nMo6U8+AP{tPC4gx!2Hu6>lrGi1z<%~*pFa9{y)df<|lWfmdOyERx+}|Ef z#0o7b{o5Qa^w^soTkVM`2b$nE9(RBOEpp4-XRo@+wl|8L-)@|-^-taR?FF|{nzR3w zx$!-nT{3TGkKC<=gOHHWJhd*11VPsI?^&Na;W(gAcJr)L;PCL1i>@&40Qh|De_X4(8!p=WLS92QnM<46$N?q8qs z`9qcubkeyN2{)_UVfBoQ4z=xyq%0bfYW>Dd@ASS(Azo8QczyVK3(OdeX6+D6x8Ks& zcGzM3H;5Kkmc+jwe5ik4cdgABRSG&R5?cKvU5OIKX92vZ%GH?%d=)RWr{enrc8H9q z&&dmk;{k?8Gv|QY4_1M#NeQr&ANu{K)crXub6Syx%K6<`i$0iIj(?^X7frQhu0nB1 zZRZ=ED(aFW;pnd+9#?*u2}=8aruWa2StyDUqOQo~<)1`{G9>aH@mcb-@lru#4l?G@z;Oqg?(mHOQQwqG0Mt6rdf7Kevp^#%c(c626Ug64@=dv1UIa`o2|b;nO%;RhZID7c?C#)o8_wjs7=@P4pXbjyqn_t zr8K8$tL{&OB^S*1t0SlJc zIi`JgO99Z?T-%4*KhBRBh8uWVVa4f5wq&&1SAuvUD{Is zWc7okd7hTGw20VX@HtF#7-2`!;j9y3@(WETGHcYPW04j7)65qB+!~@Pn!#%w1!I;r zSsflz<$#Dr(`SnGk+lU|%{NHf@33q&5{C8^PV*$P1NO4Ac^T0oQ*SdmGAPV8HMqUM z&q>Sh(8&{X@i~&kgBA$tPIoNk?!1{pnXj1*}|IX<`jI z(*B!W1idE!On9FBxw}3bgOup-yCz~;Hmah8#fU}8UhC*8A%!6 z&*u}_OVP$<>Ri~^*jPm#FrhA#XjYqkvlOJI;W4WtjU6QU%w!2x9pFrvSG{JV(P4aNin(bOEF6r8_TLt zu#V<;T^(=lBt+H0!G$;)KC%hw<3SY?yoT>>ON+)agqS@~2D&Faa$ZYcy^Z$ECGRdQ z2M_l+MOgXCMM@PLPft@x=&!G?%Jk^lrQ!?^pAm!~+6_Nb0GiYm@UKcm(B#N$4Hj>4 zT0IH7-(!HVJhkq8f8GEx#r=fU6*o9RkL~B5*?PXZaxX7zEL2(<&@spQ>*`mx{slOU zJ)pEGK3zAR`H~LoJ63SRPQ9 z4Cc>7pYr{AP=Rbt07VpR@nlIB#Jhc3aiv$@J)tR;lNkI^SqvH>8cL~hA>;$|Kqt~wSlbIBS#lYej^}WM{JO>Ezg%IWs3oa3ipH-JSSJ{Cm zJLqVgJecx&Yq&JeUzA1gl{`@W1930pkv-_(XN)0%1yH~bS`B$UE)E66p=caq?XWSBv;{ughi1b+hZW}WN3{hi2DX6C@rCn<&mM^2bZ z#)3v*Odzqx+wP1P^Ly-0rr7x(EVQuBlo7VJ0|%HfR4pJiv*j z&a5+pyk)ZKg_D(U$C$QxmP4k2GDfFHeXAqjsQ2mM;raKun&rdi?r%Ypud#09BY@Z~ zi}|2}o_+R=f+F9j{%x_|&;#mtZ%PN&N}awPSAAK0^Y#VFSQXt{<#1*iGiQ~b?}VxK zt>y5aa;YI639M?Xrg6L`h_<(PWW>x5-baF0aL~21K8ycqbeh*~cM^_M_4`y((lDc| zudl)-qnfzOU|f_Ing^f*sP8B4H?l(>@Uo#h$%_>Xqh|-Zf5`Zfo^#+JC<2pbcC4J? z@a++?L)}vgKopKULnGnCzb}v_0?|)j>RU(9hQ)mzqf`|tfFptNQyfxZy+v0OJ}SPI1u5%B?9@c;bK$zAT?S^} z*6t*G4+ySBe{hNmS<9}c&!(}H`d65pNkI~DFUpS{Ws#}W8Gk(6tfhPbUZO-;^rqOW ztAKo~OIn5cvvzs7h}&k-ug`OLlM1h@*450jhMBKtW1k6?+-PXh9#^S(inVu~OF)s< z`0ZC$dkyCoxOHb|_U~+a6f#9c?Y4q28hqvjeXOq?8m|A+^M3Hy*k~$Bf^5_cm8xYV z9nZR7*A+17ACK2GHtL_%51J_jy(oSC#Bf>q!*Jd!sn#_c_wJ2!rUC1jyUbkgjnvob zU!V~oPdL1T{asH-bxz!l1PsO)G+!2DmGYH#&Tn@tJ zzQwGWn_Wk6vhkj>8JOVJ4zg2&=aUs{=SPyMrj2PY#o+8Jp7GkdlK{I@H`7^joZUUQ ztNFq2GL=(o4w{2gDN||8FGXQYGY(ST$3^Q~MM<-+sqX@YtP1XLPhT<-ZtOk_&>VF# ze5I=2p_G1iOx3z_{N47(^g~kNa$yVNsmktr_DQAG{cCNh`+t?@KXoe23T$So3oGyc z`aV3meRa%h|7Uw(_E5F>NHddPnOx?Nr*EX8ygWcUq}?n49f%8{PPGUi>vgHJvN9A$dLj z@7dYem~z$nPeu`#0P|EtG>)tW*1o%G&f0j%)JD|DGm6sa+uPcv#fp~hr?ODfnqO+1 zOe`l?zif<6YJM#TcrH2*?0Rmk(~2@hk?habznY3}ZEN#&Ct<$~DuV01ew|q7IWl!; zzb{om7I}(_eyNq!FPD~UmU+~5i+evaIac1Xx@CM8aDyoDKk&bhsXIB*2A+uT{OkAa zp2nKP<~;H1Xi_wd8nb?*r!X5_z-{wK%HgOU?{C0bDsA+gSC2bO^GwgPQabZ^Gx&vW z{Ju9fofl1wj@DeOI|h)yY$KxmG5e8`MP_JNeh{TC`jqAr1^w#fGIV}^aVtXkWox=t z)j`-5)?oPG%Lj51bqa`vl=ubIkWigd zizKCvncU4g3@**oKCHq}&QkyCa*SN6kg8I^=A1qiBw{m?O=i5azpsArqs*g;rHMnr zV^?>`#&drz_4u~s7zgU(vlF#f`~BDZgZpdPaBPRxae5>v@APPMxsI#PW9{7T{^tSW zoH8+rmM+BX6a?A;xP&U_wmt-Bx#z;~={Bz2L~PlGoQDQ%xh^*ZP(oP(LKm`w^RMFH zs#Rg+`6W&GCa5rYAGBZD$WjySYfQ5&Sb!>Ys6~Q=92ZgaRrrA~x`;_A8nV7FRd@k3j@ zb(SE_$=37pzS~LW1J8$;zyE$IY0mAcXAIpLOxM?*bt2uVSYDJixV`(p?>V&J@Mk04 zZMSM5+%WnH=TNi+5&MJmq7pj4%g83dG?OUif8#Y|gJ-Q2mL_!8nV0(X-R}`0fzETk zC^E!!l~J+<3%fy$+U1WR4D2X}_+mkmfY~~0)*0p*R$;%3zh!i3#ii*rSGR|Ckm)I* zyNJlz_1NAUqgeu`X%A5cDzrbtvPR3=(I_QqT1qY+*|I!IaSAgM2= z()5S(=*`C&WU5$ly&&Bh0OU^ldv9OJ6_sQNfI{`1a*w|&-Jzq7a}BH;*U^Zz{f>Ca zp-kO?uN!ZN52fX6;D3C zt)UL%Cu9dm@e;AaflsEI|)(-*L?RMFTnLZ-ybZx}(+vX6AiA@sd-cMWsnY+1T5m>xp-$(8S+z5nuM+InLJ z(AmKiEvSk=JD)Qth|c%%@-5$O<4tXa67ub}=kTGB!;#m( z*q99Pgi+SB0?QdC!OAe|#mHCB2JILCE)7?(;oV{@*|S!l(9)ed_ z-Bj5cnf$(K9OYcCyDD>c2Ln0ff1HTwruEt5a*#{%aU(>#311NFb#EV#>ss{8md+I3 z1E#1Ou7Ss$dAjsv8iQdj@DB_ikw2cZS$JWxCjjg#`S=B3Yf=Q>xpLR&+PM3+D`)5) zg-mLxI(=*zwB<|@zLGeZ{Qhz*%)l&Hy29bz4F-+qY5C@Pdg_Vqz}!Na&jN>Hu8C<5 zt7RTF%cljq&@FGhBA?sS8SDLYcv(kH95@hQM&j{hIg$rYMhV7hx4e5xmbxtxrfbdf z8uu_p|KZMD`gMfj!~j%#k>7`=#iTKi(VjYB@KGP4oU4?Y+r7M<;O{RB#SXH>Tr1_h z#jTv|2W)=;pB0y=NkPOr6MHOwJTJZpXkBt_TJov1A`SbmP?*KTlVu}@Uyz_D^sH)q zm4OtDjA+c&+dO_?Yo2gIYg4N#Mu4`sStFy8@RjDKS+w~W30DdFC zgR^a6jNuta-ym$=u4^%ick?KtXQEVdw0xMD`1I_AK=4%Y&#Lzw_9GU(7!&w;^!m{0 zk5r)BN0dfz^)Z9=yKZ&M=U71);Ou;~EWSC3F9wH~#>bO9DTT@0Jv>R|7%~6y_W2i6 zmM0DNg3E_jwf@NG;!589?ZH$~+3yvFX7SPTb7oTzYOrt`9?Q!T|CA-(dtuq&J;SvR zPwh=wT%1khogTcc`GaS+qXygr9avc4;JUx^LOCEebqU}XESTH!50%&C?AYgR9^DWN z?oZ&);y}@wGK%>X1DK%c#U@sSW=;V<*YhgpcLyVmc|v5csb7xny@Z{`_*|V;79!Sz zuyP(Ne#u5C1Fmx8#PV{UR)%uLbN#+Ea-4+4cPuphy)g7s&{NytVlwW&jzN$A>Yu=BI5AYFzSw}vpL+8RyaRXcVct-X|#X* z%XHKCFo8;`PJ4$c^;u(}uFeGNnBWPB$hP2H{Lf&Qm||p0S++dN2xxwok7e>vhWu{6 zmqi&K9ms zsE54@k@*jF%pgFjfwRI#L2zGybaKF>fl;CtksYcj9#=7ulRwA1Ab%W-%&lJQwMG2g zKmVa0W6}A$qXi`($ZTR$H@h9we_vzRy|d;}Q^H~n75VkWfV0hkz)H)M0FVo!T`hi+E} zlOLd#?ZPDMN&5V|V&(H61=avL*|-xt2Kx*+@=w!$TVv+VZK-!&--r1xmwdM`3HKuU zIhg1hpX?M88#@Vv1($17t#4<|*4unPs0QxfhnQ&+y#Bf4eqW6=eT@GeP6`gpOuDjQ zVwmOQ|9EwhLTfB9e=u?;uPq=HH?UB(M*4V`gEP`nr!alNVAfHPO3L5wn-@btm6N-t zJAuyyNYKG)v;v1b^*Hed@{2@^`0o6;E9|afSSvl#Sv2xnxWRVcOedP zoJKL@Z&W4o;sGR@8|q@#>Rw)D3aBf__f7$VNWCW^oB`03;Xml&87rkBJ0{Osem?mS zcSV-YD7BU>+yf9Jx&Bu>p-1f-X~x>36R4gyP0_xU=?#s)-{1lo8nm*uO4US@!MGTJ z{1SCtSBf-YQ1au=Z6D4ODvha5>Vd4BllD{a$Q#gf2{kmXzQOJL6@~*I|M&F|bE@t8 z@`SI958SR!sfMz@qyY_`DzCeXPdjVx8~Lx-2W_lHKF5PB*872Q;0D3#A6c0-$Xg-@ zx-~u=>1t&UuB6w1fITV0jo%BB&m)kk0x=TQ#(Al( zF$uzEG&jWsm>^y9H-9k-0utcTo*V&iQyx0M4+q6x;D^5`GA1D@p8oq}w0V7c13mT3 zG>mL3(Eah70-66S;Y=K%YP$e#B$P>bTkOVYVJK_%wTw0UK1uW1Cq~(sB2pPyDr>gX z3@sxHLn$&sl*Aa>MJQw`drJDwyx%`?=gz&)Irlv0_j`^hHspD_?C-UC2l4ec*SLdg zNX7vdB<%1U@;wE_?FmMt$y=rec>DmdRX&p6I#KjVQWJJGcuDf$;nOW)6exTKg z_B~9uxzZas=79c{(p#fV!K;ZHvAvOf&oSZ=7R^i(L(k1&g0SbJ^PUu zq3h&nFhq8Y7^xiG%fwStA6AV^ptVf{@)bl;TWtK@g@qu}#+MwH z`XSdUU#r2K;;J&M15Wawfqfk0PZn}6T}|da&xK&0g*j6D`tx~BZag_ zC3J^7mu5LX@hGg>zyiLrx})n>5uQeVuF_dZiYxLbMp8N1-#kIT@)v`Hbf$>0v2Rr_ zyKxpV*8YzBf{fy+nv@k%j7f{B%>SVoC*s@MMw{0Abzivm+RfN+ySbe@E#Rz`QlW0N zy;JESJFsXJF5DO1eJaeRML@@*7^4J87R`v|x+_Xmp69;p|AT&p$Y*7D-zwh(N&@zf ziTHs+BUvHoGqT1pji}Y1e|ky~p9kwbzAbG$)DQqRa5P9!0_Jo8@Q)8__!IF_yAIgz z^!&3?=hgJHAS&d(v%_r>IJPk^k*ihm^72$@F9MW=r^{b1yd&d%d=ifBg7V4Sx%8Mk zHO16eKvp>ErlR=$cv~{ixDX!Bo{4pbNJjv5>>p&!mZWLUq-@o#!irixj;0?ig zOg}7q;GC>6AhOH1+HSVT@Le{9148te-w_{GW9xJQN&e@~-S}u@+8k0gALwZG zfOYz4c2x~d!A&Fu&=J6jZb(2X>e;|5Nn!zgs^-o-p-;- z@9tvrurWsbWAeyENYdM6LGJT6de_u34B_9ZoKb>tgREu&poiwSjdRbJM3359aGx9rs6*?;$P#q0lB zVbSH0HkS+*wNEQ_U#3qx9K$gH9j^rbYhyMo(0(qRqdsPvYibSGCy<^osx8}uZ1!x$H^V5p#GvJFvq_$=V+g!P3HI`BNw+JN88EmO&R zBC5U*L$3PCz)rse7-_b`PRk`w-_F%7a*u?utakJrb$yB=sszaHK|5HbPIvX6gGBwH z`pCXrQO)uq6vk{A?N=+BGM*pBq+ojn2WdYgVDn249H^u-w^5zm+>o-F?Jc9GrS<4L zFLK0X{yMOz0zEgnWS`jqq1heprS)mmdi!<>1n=j3sJ%@KXcaKO0Mb3oabt@7AB|PRU8!F36d2E^MwE!G6%=^*vj&;m}na*uiIs8Cb-%e(F>- zwgndL(6e)${kEV(uouasMU`K&F}gWey#B{)KCPjgls!BGsp2eo zmqTF%3WsvFLJz^^Zl#z|F^QHCG0s=6=hTKyVN96))T>%rpg}-F--$AmX?4ZzFKfh` z4WRrmU%q@<>2a{R)rhzCu?0c}FOATeCPDXSqMekJn(Pi)7nnhNwFGPgn^i;<5G|_d zjc!=^iJQMwec#_LSG$T6`!>4+L7v)>`tJd7Kiz|9yYI=+N^8&MnOk`I{r<>H82NL& z)m+X?HTNXluTv8D>Y^417^g$K53m)Oj$GA5c;Z+desT7Qqm(p+!alzLQqar!nu6g@ zSd}bp{q@z=mXJz%O)CY#R#T^=rHVvGby?QYIS|T7Z(J!nA zfpp?w{XO(eiU=WRRKLjPv9$1KO&BCJ@!V37(IMaz%`^M$+mh$=i8ra#mgN1c zS24dGZ|O=v0T~_^W_$XyM4^669y?f|T%i`~L_)uFFZ&I#_wZ;i1n+^O_n%z?>k5*H zvS*52XsBwfd->zG3?v9*IuUbiZ|!X?zJ4gok5EWx#6ONhdIi?>M!05<2|mHbp;~X# zikra+Zuzyp_9m3aEU*h^Cf{cC^z~_Yd-V!|$i^?Ff3j0=-Mv@&@>w4U5`y2~^O`4* zF`NE{0TR7yEOsS1zt_JXF0?jdyFeK-RUmql^6D7sdda@O?tg$p|DOXrCV`lmyleS{PG{U=t%*(I4d2cO+; zfs_gZrRG>JRN_eab$1A&wYh;48lxAhzM8na4*}qWQJ+jpY@JB@az=XG(h>9Skse4% znbL!Tae4XqP7h{u3az1wU-6aOHPGI}!x~bS!~FALup~vAIKAfd%jO?l(ft2}6Bs4s zujV_MssoJ1^Xv)po^}d0xh)V5YP>}8dRMqMlXoh5Jw6_wgE8V{*GA&w++3)%3c8&1 zA}=)Ezz3N+CQ7;e1;Wx=tWffXAxhwTvA=AVmKegY`7Q6*2@VwvjR(YHf!#%&raF-e z@7=`RE83wLo`Db%X%UTxjLb-zFt)EB=<536R7SkdmDROvlM8|5H6-0}*~^`E@+rX8 zw`;?q^C8fc^*i|x(zzA$=sL4xG!P@PnWn1-=TIdDb5PI)Vc$1qPUHqGBB@K8!A4Y# zYo(5kj$Ku-1%@;(GIP`asWSfkEYJDM%wKj~_>|Lc1X4YfMIW$oovIHcw;YEU5(y{# z3Z42;qIY4g5g`v+lBcbMNw1$Nv0`YIm*TpaJJfy!P}_*CV0n239!ABk$k{biAcEoB9w;^c)H zh+Fv8@Ng~j-{)VBtSq)lv0%meZ*6WwmriLoiWtP|9>LMcK!#0pJ9lfLG+{^36m%o{ zx4_+0#IVCoOVJXT$-g=APt`&^3gxSD$ioJ-QJ|(u0{{ zJHN_R%jlPQjlPPZhtE2*i|-kbLGCyFnLif>%}(aZ42!LztAV8q%^gmw+e-04kf-}c z*?Hpy^il|m<1KX0mYr<59Li2+V*<}|ULKm4b9HUa-s?nDuHK%wtZlc?XlG!IgxG>6 z5-)KANj711bs6d%&G8NIOXSK8(K9rpbJpzZ^2WoC&*UdX*1TPb8Nc^miBc>KX1a zK3({?p?*0s6-^%3#1Qs~{f+N;>vzMEofw#`LQm<-m!RXHRgFhwU5ew)HsZy(Hhj+? zYG0V!9hy`-{A74$1kPA238_KxAC*Oqnm$HTo5kpn@`M0RsG z%x@1Dg=+nf`D>47?Y|lkF#%OI#T$vkyo_tgzV#=KGxu0qPI_X!t+03Go|d3Tm&DZh zq+$YUT}ms0quLgjf)kB#R3Bn!LKWpCOb_Q;XBE*@#xt;H$Q;@E@4eOku?BcYVP5Z?2!x5{F zsI$Y1p>1#Jk(&+XypwFfa@nco)V%%24u;NlkJ4{^PBD*Zk(jJ^t0>;1`}Mx3f4Ru> YTP92X36kd(%-rNK$66bgAID$*AD&OFcmMzZ literal 0 HcmV?d00001 diff --git a/catalog-rest-service/src/main/resources/ui/src/constants/services.const.ts b/catalog-rest-service/src/main/resources/ui/src/constants/services.const.ts index e1247825db2..7cbb98d83af 100644 --- a/catalog-rest-service/src/main/resources/ui/src/constants/services.const.ts +++ b/catalog-rest-service/src/main/resources/ui/src/constants/services.const.ts @@ -36,6 +36,7 @@ import snowflakes from '../assets/img/service-icon-snowflakes.png'; import mysql from '../assets/img/service-icon-sql.png'; import superset from '../assets/img/service-icon-superset.png'; import tableau from '../assets/img/service-icon-tableau.png'; +import trino from '../assets/img/service-icon-trino.png'; import plus from '../assets/svg/plus.svg'; export const MYSQL = mysql; @@ -48,6 +49,7 @@ export const ORACLE = oracle; export const SNOWFLAKE = snowflakes; export const ATHENA = athena; export const PRESTO = presto; +export const TRINO = trino; export const KAFKA = kafka; export const PULSAR = pulsar; export const SUPERSET = superset; @@ -74,6 +76,7 @@ export const serviceTypes: Record> = { 'Oracle', 'Athena', 'Presto', + 'Trino', ], messagingServices: ['Kafka'], dashboardServices: ['Superset', 'Looker', 'Tableau', 'Redash'], diff --git a/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts b/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts index 5d6353c8f28..6d73ce065e2 100644 --- a/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts +++ b/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts @@ -33,6 +33,7 @@ export enum DatabaseServiceType { MSSQL = 'MSSQL', ATHENA = 'Athena', PRESTO = 'Presto', + TRINO = 'Trino', } export enum MessagingServiceType { diff --git a/catalog-rest-service/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts b/catalog-rest-service/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts index 8e611a2210f..fbaa2b59a83 100644 --- a/catalog-rest-service/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts +++ b/catalog-rest-service/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts @@ -76,4 +76,5 @@ export enum DatabaseServiceType { Redshift = 'Redshift', Snowflake = 'Snowflake', Vertica = 'Vertica', + Trino = 'Trino', } diff --git a/catalog-rest-service/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/catalog-rest-service/src/main/resources/ui/src/generated/entity/services/databaseService.ts index 0ee0eb186ff..abc200c0eb6 100644 --- a/catalog-rest-service/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/catalog-rest-service/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -94,4 +94,5 @@ export enum DatabaseServiceType { Redshift = 'Redshift', Snowflake = 'Snowflake', Vertica = 'Vertica', + Trino = 'Trino', } diff --git a/catalog-rest-service/src/main/resources/ui/src/utils/ServiceUtils.ts b/catalog-rest-service/src/main/resources/ui/src/utils/ServiceUtils.ts index 96305e6afcf..ae883f7456f 100644 --- a/catalog-rest-service/src/main/resources/ui/src/utils/ServiceUtils.ts +++ b/catalog-rest-service/src/main/resources/ui/src/utils/ServiceUtils.ts @@ -23,6 +23,7 @@ import { SNOWFLAKE, SUPERSET, TABLEAU, + TRINO, } from '../constants/services.const'; import { DashboardServiceType, @@ -64,6 +65,9 @@ export const serviceTypeLogo = (type: string) => { case DatabaseServiceType.PRESTO: return PRESTO; + case DatabaseServiceType.TRINO: + return TRINO; + case MessagingServiceType.KAFKA: return KAFKA; @@ -206,6 +210,7 @@ export const getEntityCountByService = (buckets: Array) => { case DatabaseServiceType.ORACLE: case DatabaseServiceType.POSTGRES: case DatabaseServiceType.PRESTO: + case DatabaseServiceType.TRINO: case DatabaseServiceType.REDSHIFT: case DatabaseServiceType.SNOWFLAKE: entityCounts.tableCount += bucket.doc_count; diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 73b61b1bad1..ab1b464b1de 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -83,6 +83,7 @@ * [Oracle](install/metadata-ingestion/connectors/database-services/oracle.md) * [Postgres](install/metadata-ingestion/connectors/database-services/postgres.md) * [Presto](install/metadata-ingestion/connectors/database-services/presto.md) + * [Trino](install/metadata-ingestion/connectors/database-services/trino.md) * [Redshift](install/metadata-ingestion/connectors/database-services/redshift.md) * [Redshift Usage](install/metadata-ingestion/connectors/database-services/redshift-usage.md) * [Snowflake](install/metadata-ingestion/connectors/database-services/snowflake.md) diff --git a/docs/install/metadata-ingestion/connectors/database-services/trino.md b/docs/install/metadata-ingestion/connectors/database-services/trino.md new file mode 100644 index 00000000000..a20221dd4d5 --- /dev/null +++ b/docs/install/metadata-ingestion/connectors/database-services/trino.md @@ -0,0 +1,95 @@ +--- +description: This guide will help install Trino connector and run manually +--- + +# Trino + +{% hint style="info" %} +**Prerequisites** + +1. Python 3.7 or above +2. OpenMetadata Server up and running + {% endhint %} + +### Install from PyPI or Source + +{% tabs %} +{% tab title="Install Using PyPI" %} + +```bash +pip install 'openmetadata-ingestion[trino]' +``` + +{% endtab %} +{% endtabs %} + +## Run Manually + +```bash +metadata ingest -c ./examples/workflows/trino.json +``` + +### Configuration + +{% code title="trino.json" %} + +```javascript + "source": { + "type": "trino", + "config": { + "service_name": "local_trino", + "host_port": "192.168.1.32:8080", + "database": "default" + } + }, ... +``` + +{% endcode %} + +1. **username** - this is an optional configuration if you are using username/password with trino. Please use these fields to configure them +2. **password** - password for the username +3. **host_port** - host and port of the Trino cluster +4. **service_name** - Service Name for this Trino cluster. If you added the Trino cluster through OpenMetadata UI, make sure the service name matches the same. +5. **filter_pattern** - It contains includes, excludes options to choose which pattern of datasets you want to ingest into OpenMetadata + +## Publish to OpenMetadata + +Below is the configuration to publish Trino data into the OpenMeatadata service. + +add `metadata-rest-tables` sink along with `metadata-server` config + +{% code title="trino.json" %} + +```javascript +{ + "source": { + "type": "Trino", + "config": { + "service_name": "local_trino", + "host_port": "192.168.1.32:8080", + "database": "default" + } + }, + "sink": { + "type": "metadata-rest", + "config": { + } + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} +``` + +{% endcode %} diff --git a/docs/openmetadata-apis/schemas/entities/databaseservice.md b/docs/openmetadata-apis/schemas/entities/databaseservice.md index 1f65df0418c..650dade7222 100644 --- a/docs/openmetadata-apis/schemas/entities/databaseservice.md +++ b/docs/openmetadata-apis/schemas/entities/databaseservice.md @@ -48,7 +48,6 @@ Type: `object` 9. _"Athena"_ 10. _"Presto"_ 11. _"Vertica"_ - - + 12. _"Trino"_ _This document was updated on: Thursday, September 16, 2021_ \ No newline at end of file diff --git a/docs/roadmap.md b/docs/roadmap.md index e94a80f2ab7..048e86937bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -56,6 +56,7 @@ our roadmap yet, please file an Issue [Github](https://github.com/open-metadata/ ### Other features * Data quality - Data profiler integration work in progress * Schema versioning +* Support for Trino ## 0.6 Release - Nov 17th, 2021 diff --git a/ingestion/examples/sample_data/pipelines/pipelines.json b/ingestion/examples/sample_data/pipelines/pipelines.json index d1c83160aa5..3b834334309 100644 --- a/ingestion/examples/sample_data/pipelines/pipelines.json +++ b/ingestion/examples/sample_data/pipelines/pipelines.json @@ -6,6 +6,13 @@ "pipelineUrl": "http://localhost:8080/tree?dag_id=presto_etl", "tasks": ["presto_task", "assert_table_exists"] }, + { + "name": "trino_etl", + "displayName": "Trino ETL", + "description": "Trino ETL pipeline", + "pipelineUrl": "http://localhost:8080/tree?dag_id=trino_etl", + "tasks": ["trino_task", "assert_table_exists"] + }, { "name": "hive_etl", "displayName": "Hive ETL", diff --git a/ingestion/examples/sample_data/pipelines/tasks.json b/ingestion/examples/sample_data/pipelines/tasks.json index 0e7f9f68f2f..28d17249a95 100644 --- a/ingestion/examples/sample_data/pipelines/tasks.json +++ b/ingestion/examples/sample_data/pipelines/tasks.json @@ -30,6 +30,14 @@ "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", "downstreamTasks": ["assert_table_exists"], "taskType": "PrestoOperator" + }, + { + "name": "trino_task", + "displayName": "Trino Task", + "description": "Airflow operator to perform ETL on trino tables", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": ["assert_table_exists"], + "taskType": "TrinoOperator" } ] } \ No newline at end of file diff --git a/ingestion/examples/workflows/trino.json b/ingestion/examples/workflows/trino.json new file mode 100644 index 00000000000..5749a98117f --- /dev/null +++ b/ingestion/examples/workflows/trino.json @@ -0,0 +1,29 @@ +{ + "source": { + "type": "trino", + "config": { + "service_name": "local_trino", + "host_port": "192.168.1.32:8080", + "database": "default" + } + }, + "sink": { + "type": "metadata-rest", + "config": { + } + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} diff --git a/ingestion/setup.py b/ingestion/setup.py index 7c02dc5eaac..0f03004bdae 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -98,6 +98,7 @@ plugins: Dict[str, Set[str]] = { "oracle": {"cx_Oracle"}, "pii-processor": pii_requirements, "presto": {"pyhive~=0.6.3"}, + "trino": {"sqlalchemy-trino"}, "postgres": {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"}, "redash": {"redash-toolbelt==0.1.4"}, "redshift": {"openmetadata-sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"}, diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index 214b7ad3228..0781afc883e 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -23,6 +23,7 @@ class DatabaseServiceType(Enum): Oracle = 'Oracle' Athena = 'Athena' Presto = 'Presto' + Trino = 'Trino' Vertica = 'Vertica' diff --git a/ingestion/src/metadata/ingestion/source/superset.py b/ingestion/src/metadata/ingestion/source/superset.py index b3f7992eeef..1ce66482c95 100644 --- a/ingestion/src/metadata/ingestion/source/superset.py +++ b/ingestion/src/metadata/ingestion/source/superset.py @@ -64,6 +64,8 @@ def get_service_type_from_database_uri(uri: str) -> str: return "snowflake" if uri.startswith("presto"): return "presto" + if uri.startswith("trino"): + return "trino" if uri.startswith("postgresql"): return "postgres" if uri.startswith("pinot"): diff --git a/ingestion/src/metadata/ingestion/source/trino.py b/ingestion/src/metadata/ingestion/source/trino.py new file mode 100644 index 00000000000..01d593bb36f --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/trino.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from urllib.parse import quote_plus + +from .sql_source import SQLSource, SQLConnectionConfig +from ..ometa.openmetadata_rest import MetadataServerConfig + + +class TrinoConfig(SQLConnectionConfig): + host_port = "localhost:8080" + scheme = "trino" + service_type = "Trino" + + def get_connection_url(self): + url = f"{self.scheme}://" + if self.username: + url += f"{quote_plus(self.username)}" + if self.password: + url += f":{quote_plus(self.password)}" + url += f"{self.host_port}" + if self.database: + url += f"?schema={quote_plus(self.database)}" + return url + + + + +class TrinoSource(SQLSource): + def __init__(self, config, metadata_config, ctx): + super().__init__(config, metadata_config, ctx) + + @classmethod + def create(cls, config_dict, metadata_config_dict, ctx): + config = TrinoConfig.parse_obj(config_dict) + metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict) + return cls(config, metadata_config, ctx) diff --git a/roadmap.md b/roadmap.md index d7a7cdedb9e..a23990020f8 100644 --- a/roadmap.md +++ b/roadmap.md @@ -57,6 +57,7 @@ our roadmap yet, please file an Issue [Github](https://github.com/open-metadata/ ### Other features * Data quality - Data profiler integration work in progress * Schema versioning +* Support for Trino ## 0.6 Release - Nov 17th, 2021