From 6f691cbc925ee837b203ff9c2bfa99e63a394b8d Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Fri, 8 Jul 2022 10:28:59 +0200 Subject: [PATCH 01/23] Update google-cloud-bigquery to 2.13.8 (#225) --- build.sbt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index afed95e2..d4252c77 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.13.1", + "com.google.cloud" % "google-cloud-bigquery" % "2.13.8", scalatest % "it,test" ) @@ -70,11 +70,12 @@ lazy val cassandraDependencies = Seq( ) val circeVersion = "0.14.1" + lazy val jsonCirceDependencies = Seq( - "io.circe" %% "circe-core", - "io.circe" %% "circe-generic", - "io.circe" %% "circe-parser" - ).map(_ % circeVersion) + "io.circe" %% "circe-core", + "io.circe" %% "circe-generic", + "io.circe" %% "circe-parser" +).map(_ % circeVersion) lazy val scalatest = "org.scalatest" %% "scalatest" % "3.2.11" From bc6403953d881604689a2a53d44f1c14a2532fc3 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Sat, 9 Jul 2022 20:22:35 +0200 Subject: [PATCH 02/23] Update spark-core, spark-sql to 3.3.0 (#227) --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index d4252c77..9a0b2185 100644 --- a/build.sbt +++ b/build.sbt @@ -58,8 +58,8 @@ lazy val bigqueryDependencies = Seq( ) lazy val sparkDependencies = Seq( - "org.apache.spark" %% "spark-core" % "3.2.1" % Provided, - "org.apache.spark" %% "spark-sql" % "3.2.1" % Provided, + "org.apache.spark" %% "spark-core" % "3.3.0" % Provided, + "org.apache.spark" %% "spark-sql" % "3.3.0" % Provided, scalatest % Test ) From 671a0935b677ea41b399fdf3e2ce39d7845e14e1 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Sat, 9 Jul 2022 20:23:03 +0200 Subject: [PATCH 03/23] Update sbt-scoverage to 2.0.0 (#230) --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 3c2f8ba3..e6427e45 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.13") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") From 7e376d7467578b4078d8fbcbe4dff62cba2d30a9 Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Sun, 10 Jul 2022 12:19:53 +0200 Subject: [PATCH 04/23] Removing Scala Steward GH Action --- .github/workflows/scala-steward.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/scala-steward.yml diff --git a/.github/workflows/scala-steward.yml b/.github/workflows/scala-steward.yml deleted file mode 100644 index f472a7dd..00000000 --- a/.github/workflows/scala-steward.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: - schedule: - - cron: '0 0 * * 0' - workflow_dispatch: - -name: Launch Scala Steward - -jobs: - scala-steward: - runs-on: ubuntu-latest - name: Launch Scala Steward - steps: - - name: Configure GPG Key - run: | - mkdir -p ~/.gnupg/ - printf "$GPG_SIGNING_KEY" | base64 --decode > ~/.gnupg/private.key - gpg --pinentry-mode=loopback --passphrase ${{ secrets.GPG_PERSONAL_PASSPHRASE }} --batch --import ~/.gnupg/private.key - env: - GPG_SIGNING_KEY: ${{ secrets.GPG_PERSONAL }} - - name: Launch Scala Steward - uses: scala-steward-org/scala-steward-action@v2 - with: - github-token: ${{ secrets.GH_PERSONAL_TOKEN }} - sign-commits: true - signing-key: ${{ secrets.GPG_PERSONAL_KEY_ID }} \ No newline at end of file From fc6d8cf1878ead0cee32fe1929eec2cae7db872d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Mont=C3=B3n?= Date: Sat, 23 Jul 2022 15:53:17 +0200 Subject: [PATCH 05/23] Documentation improvements (#231) * text fixes + logos * closing image tag * Scala 3 tag for 3.x * inline styles in js way * resize images * background color for some images --- README.md | 16 ++++++++-------- docs/Modules/Circe.md | 19 +++++++++++-------- docs/intro.md | 14 +++++++------- website/static/img/logos/bigquery.png | Bin 0 -> 2989 bytes website/static/img/logos/cassandra.png | Bin 0 -> 6304 bytes website/static/img/logos/circe.png | Bin 0 -> 2428 bytes website/static/img/logos/scala.png | Bin 0 -> 1826 bytes website/static/img/logos/spark.png | Bin 0 -> 5548 bytes 8 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 website/static/img/logos/bigquery.png create mode 100644 website/static/img/logos/cassandra.png create mode 100644 website/static/img/logos/circe.png create mode 100644 website/static/img/logos/scala.png create mode 100644 website/static/img/logos/spark.png diff --git a/README.md b/README.md index 04aebcbd..b71abc2b 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,14 @@ Check the [Documentation website](https://data-tools.github.io/big-data-types) t # Available conversions: -| From / To |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | -|--------------|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| -| Scala Types | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | -| BigQuery | | - |:white_check_mark:|:white_check_mark:| | -| Spark | |:white_check_mark:| - |:white_check_mark:| | -| Cassandra | |:white_check_mark:|:white_check_mark:| - | | -| Circe (JSON) | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| From / To | | Scala Types | BigQuery | Spark | Cassandra | Circe (JSON) | +|:------------:|:---------------------------------------------------------------------------------------------------------------------:|:-----------:|:------------------:|:------------------:|:------------------:|:------------:| +| Scala | | - | :white_check_mark: | :white_check_mark: | :white_check_mark: | | +| BigQuery | | | - | :white_check_mark: | :white_check_mark: | | +| Spark | | | :white_check_mark: | - | :white_check_mark: | | +| Cassandra | | | :white_check_mark: | :white_check_mark: | - | | +| Circe (JSON) | | | :white_check_mark: | :white_check_mark: | :white_check_mark: | | Versions for Scala ![Scala 2.12](https://img.shields.io/badge/Scala-2.12-red) ,![Scala_2.13](https://img.shields.io/badge/Scala-2.13-red) -and ![Scala 3.0](https://img.shields.io/badge/Scala-3.0-red) are available in Maven +and ![Scala 3.x](https://img.shields.io/badge/Scala-3.x-red) are available in Maven diff --git a/docs/Modules/Circe.md b/docs/Modules/Circe.md index c5a8a301..9ceec6ac 100644 --- a/docs/Modules/Circe.md +++ b/docs/Modules/Circe.md @@ -5,7 +5,7 @@ sidebar_position: 6 [Circe](https://circe.github.io/circe/) is a JSON library for Scala. -The Circe module of this library allows to convert `Json` objects (from Circe) to any other type in the library. +The Circe module of this library allows to convert `Json` objects (from [Circe](https://circe.github.io/circe/)) to any other type in the library. :::caution For now only conversions from Circe to other types are available. Other types to Circe are not ready yet. ::: @@ -16,13 +16,16 @@ but more specific types like `integer`, `float` or others do not exists. Because of that, any conversion between types will convert `number` into `Decimal` types, as `Decimal` is the only one that can ensure the precision of any arbitrary number ::: -
About Circe and private types
-

-Circe has more specific types than `JNumber`, like `JLong`, `JDouble` and other, -but all of them are private to Circe itself, so we can not use them, not even for matching types during conversions. -In any case, even if we were able to use them, when parsing a JSON string (probably most of the cases) -we can not detect the specific types -

+
+ About Circe and private types +

+ Circe has more specific types than `JNumber`, like `JLong`, `JDouble` and others, + but all of them are private to Circe itself, so we can not use them, not even for matching types during conversions. + In any case, even if we were able to use them, when parsing a JSON string (probably most of the cases) + we can not detect the specific types, we could only guess them from the data. +

+
+ ```scala import io.Circe.Json diff --git a/docs/intro.md b/docs/intro.md index fbccb59e..c86cd953 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -33,12 +33,12 @@ or a BigQuery table into a Cassandra table without having code that relates thos ### Available conversions: -| From / To |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | -|--------------|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| -| Scala Types | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | -| BigQuery | | - |:white_check_mark:|:white_check_mark:| | -| Spark | |:white_check_mark:| - |:white_check_mark:| | -| Cassandra | |:white_check_mark:|:white_check_mark:| - | | -| Circe (JSON) | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| From / To | |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | +|:---------------:|:---------------------------------------------------------------------------------------------------------------------:|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| +| Scala | | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| BigQuery | | | - |:white_check_mark:|:white_check_mark:| | +| Spark | | |:white_check_mark:| - |:white_check_mark:| | +| Cassandra | | |:white_check_mark:|:white_check_mark:| - | | +| Circe (JSON) | | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | diff --git a/website/static/img/logos/bigquery.png b/website/static/img/logos/bigquery.png new file mode 100644 index 0000000000000000000000000000000000000000..a89625c80ae418f29e9636821a10b11c813435a4 GIT binary patch literal 2989 zcmZ`*X*ksH_x{K-*$%RkuXCU0eqNk&u5&%f)|SS+T%ue60PvccU~QPF_MaW+ zV0uZtq`Cu$C08pR7&2VF9#?l@pHWmO7brt|(?gGFblZyEb05=c-u;>Z^=tlq` z9Gu^Z(`F_(+|7-#z|nuFsJ%Rk$#4dnIEDfMx4?hK0zA%#GD%RFsl^4*Jcy51LwK;x z%n1Ov@=dWA+iTNPkNtw3?I3-3{GRrGvIvw6FdRxKE4Gm8wIL|fSc?>?*GLz!=dIPC z+{+gcv$H$;+^>r&kUS|w7CnS~8rmMKfc7X&f(upza1hRuOHIu$4WIc$JKK^GI^aS? zHCbdne{Z3-Rqwss-~KPZePHh;dHc#y`{2DJf&1=(E-inw6c;a-Xi`v;&?XeS5173U ztocEvxA3Oj0+HpP0m&Y%2&+BM%X=)J{BWaH~EyavSrq9Pf`96!P4qTGXwRpQ#lH5_G zIet9ECZIFa$;PLBzd9J+?g1BNM|ET*waiV%0+}DN{V>u>39hhdraod_->gd5gJJVb zdh{-J9+f@tga5lYd&^l`F3!IFW{7_6PC!csxl{rdH zVS;=;=Jsyle6(C=mT184P=;04mC>M-M*AUKmBQK(?#D-vNuoSVnpCis?aBV3hp5E< zvV4Xp)pTDTDNJTnsH!y(=Tn_O8_T;89l)62ACvB#JVP>e-1?1+ytLcCH%-LiW6-MJ zAZp6mrCody9q0EFG;U`Y9`FKs{&eo0LurBx!}cV&1+oKYVd>-ow|ujb#fe?tUEm|} zp26V~X=kZ?-Z?Vs@8jgo>IhySBg*pKW2kZ)@p*HkC(=0FIdorJIwUI3-k&LpEQML|tKnxM+QK;+JN^Wfg_)1*`wY42x;uzlxIhUFOSsI(oaWki0X#L`&)>3#n=b7G;+{(9O^6F2C)D|11K zp_50Ra1aU9U!~c0WE1a0F}SXn(y9#c+-SV2V5~ub%5}Bw4^{E+-mG6E3+42KN0ik( z*FTiJ8G3aiA#IX7vUWCojQ>2xINDHhD*J5KZkr{J@6RxYe7m2n(R>|MlaX-8fw)dwLhuWEs(`;k$%aWLsfjk zk|{idZg*G(_V?+PtyP(PgfF6?Iaq9$x-6{5hb zr$hSa1;GV%`(zA(r?L@#tf=AZY|F`WzlwfSZ8?>ZMUI%7B0;3(xr=^@!|Zm8-l*tj zEpZU$686SKFUM_&YbwL%+PLBseTF`d4|6+UWc(3qnvW`ALh63{SJ}o>(-QzpMHp|- z$jD_rsgZY##V|!knw{4-%E^w+X%Y2-ZT;-M#b&?oQW&P9+qSKl`YOKT+R*a;)eFl4 zS&lN8=i(^I^1ctfY}sB3109)Jx4|U^lOn8gQ3ft6Cn}VY)&3WOx}z;|%x2KtM*ciE zgvVI%c5>MM;bn_R=d8-{M9e>k1SEFK*r0XDY#9G^KJ|h4(uv*amr+_H64J*=;Iy89 z9n+B2&&YQ1C9Z27@}|s71BXeuC%lZ8>6Nbm@9Lpy=6rR}-LP2vs_txTI=6efBAvgM zJuf%|65DY~*6!FxF}%>XUFNfqKlBqyoKG3K8h!XYBC<|!A~B=B)?*`WvhU~pDw(Z) zeH0lpv+^+b`lY5BohH=$K4%QcKZHvjnL&1gJ z&$Lv8;Yq`DSSyn|z3c16;;-@Z6 zkA$uYx=lLWo~#AeLKe0jE*-c+>~7U0?Q&YX@Ib2&`!&WJ~?L7+Qg#tb8unbgN@tvD#~til#@C6D@d8S)u;$+ z#?j|N+TcH=Pua(x=%=ZoCtrngi0eT}-FG4W$A4#7?@N%2(?bkj9UgZ5aMcW2B6W3! zfE(jY|MGk2`n)h@$2+XlH4pgMNA_E^4ue4x8*M{5w5K|!?6X@Mwds?ntW@ELtAo>- zLH;Y=L4Ne=`#AElp>yKQe$LY9!5ruu&&*Bl#7E0PeekNX=ON+dh_Q{^*w;ebST^k4 zELN4bzVGWoS2{(a0t52n%9S3;1h<`^jV*88o$Pf~A6Qd>fqN(Nj9NB5;JTA-Z&t=e zXLkPHzd912lMH1=Z!nw=h`<8`K0#jyT!V< z%cQ4Qbz~<-L7IEe(=(toceq8ESh5bZN6ugUS~^vt15?N8VJI z2wD`0FOd5zpE17bQ}k9uvpUJ+3FY8)+?p9btTO$f=+_U4KFTInYz% z<&3iEaHTj}wsdda*Jv^CCaLMIbms}ZJ!zLe{mI5`MUhU%#i@j>qz#_oRidZbeO&su z+Ek60U#`Z=!8@N`m+}DzVM6Xy(vsYD0f;jpo~@2psgGp?gVaT*scAAe;%cm(&a3(I zsiPek!*Wtu&Az|cn}wBn}p0D7~= z*5m!*F$9P(lSuW@cVD#jWV;|Ga8ZT2IfXt-NtHV4>RmHGYJR8#UBm>$kh|cRr}^mP zP?b;b!KJ(D?%j;xgJqbN#8`d&I_Gu@T`R=y*YFGp||K5 z;w^G`c>DW@Jv-1#243T;B`IjW=a91Y?%<3Kxq=tRtHj@EnQ6|i;AIxX$yGLyiO3CC z=TgHFuURd06}+8zORFU{`^zQA^=|?!4!1`2Rv$nr$ZLV0Pyz+$TI}vAZ#+V{?_M!| z3~4oU3Pn_4Dh<#nx}dkKWh*vsHaRH%SAP_k)t3*JIJm;#dQr?;UEx$~AD%BdGTjxn zJlf^~ixl9QUsu_%3l3qPZed<%j}R{=0BQ&|H5lR?OjW~H6^YhBqE(Sf2m~5|7+P$| f`9FeyAWvWK$p2qZ(!{066ac0dEU~o)u5teZ!qSdp literal 0 HcmV?d00001 diff --git a/website/static/img/logos/cassandra.png b/website/static/img/logos/cassandra.png new file mode 100644 index 0000000000000000000000000000000000000000..49434aef5ed1763fc4e50d957a395d544fd278a3 GIT binary patch literal 6304 zcmV;R7+>d!P)001%w1^@s66k{gi00004XF*Lt006O% z3;baP0000WV@Og>004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00009 za7bBm000&x000&x0ZCFM@Bjb+24YJ`L;(K){{a7>y{D4^000McNliru<_8ZAHXL(L zzz_fc03~!qSaf7zbY(hYa%Ew3WdJfTGBPbNH!U$WR53F;H8eUhF)J`IIxsLPCch;B z001R)MObuXVRU6WZEs|0W_bWIFfuYNFgGnRHB>P(IyE#pGBGPKFgh?WSbGPV000+{ zNklQRoq?mqM)L#xL###7c3yU z7woe4A|i;e6j7=m#Uv0y5+H^2-X_VEOtSBDk}r-S2?5c&f86Ky8!|KBeCIpoInVo^ z_kGX7p+$iIM~~px_V-^B&CSi|ZZ3IUgZ+D>(TJv|rnXO45C1Y`X~ghvV`C$%Rx9%J z^V_Zh7K`O4gj~3A0kt(X>U~{Zow|p}%ge+0bLZgX)5w$dNm!b^CrEHu!_kfB_%`V81Jlp8%{b`tpE69Macer`>KC2?%lg_%PqH{MwIpOM;~F}z=5c#t;Kt5*J8?)DF_S-f=;JXMDNS9kEW^$jQ~Efe+RVe1#0aN^ka zaC37LnjF;Z*Va@+=jsNtsnx8c+iib;>~uh2AmW68v!a;j*f>m|HVs?0Y{rBM6X56P zhk=6zAu1{g>p%SzzVdnY?Ac0nt5>haopa}+qP!fbsi|1_;Dh-4`t=wvU;svs8PnD} z6v^4y*@|RNl7?TLtR|ZcMa3mJotBQ2loX_&NkdUV9*m*{n<%PT9`7I|xi~q(Mjs-i zv^YAU#X(F?%#MGhlj78DaI`haRq9Xv71JesdJj)DSuODL@`A6gFGh?Sg?sP4R}n=e zE?&GCd1B%#SFFJ3(W8~ZHf-2{C!Tx~C#63nVt#-8)1Tn%?5z4rv;9KUOTveU+6Yoa zh5ce=m5O>!{O|(~9ylmIB}Hbl3{4!flM9-3dRTNGXwiEjEYKg}ZVsrfsYQ0Z4UO(T zXmayJi<5L-h*D%JqAYMRR>89}4}-$|;o_=CNm&_ct14k?s8__xh|q7~VCY?S%6UJ# z>{%2S6=BVqHHa3aQDxh=ZNtAWSfEtA_uFsr)HBZ@G&EH8IV&p*e|z^`#KpzocS1OQ zWUWjpiQ;5jRkCcNqRh-pd@VkO4!W?g5LO2#m_2;q?B#YS$H(_Ze2@>! z7c#LYyA0X-5P13ep+Oksn&+?AA+M$ZP9h)GMb2)E4X#!bChLtD9-v29bUeyRij~7I z$jwGselBV&N>yJb&YA;1Hz$0(;~%hz3aPqcQPGU))0G(?K71IDJ@yy|CnhR2abfuM zlTVP)w=b4H^%OikJ&~K6t4x+Eqd8s`vPjaYQ>XByIO0Qx4#6U`+$$;;hLCXVt#v|$ zAq)|&%`kfSqtN1rNnxJ&-@7NnQ519Y-@d{*n;yYldic0H%fMRT|;YRm9IGi%a@v z;sf@{iXygwxPe13Y|J>s_3e*uax1X+cG`h@U}xf+Jqt zdkSBq<|w5&TN^Rk*9Kcy9PS?wg5_KGBB`bsQxf|k%GVQm7bhf!c%xsS0bZi8ABw7B zwKZeq;je zicut_oliN2?2{*i#FF;4wIjT}z2WZeuKe1ZIdhbvxJujOvdQ8M9T4WDl+~Ls zH6{oXBLfj;aDz)*;K^R4m&?qXt@toG2ZwV?@$?{n)D&I7f9F_`VROf$qhfK#h<-v+ z3#vp}mLCIriMB0{{}rQOz0Lz|?oAQ2n3ZQY_$o08&U)dpjz90?ETT6%(R$zuHR)|+qQtvB9K z@yB1De;#urCxZlyjiQjwK(hE^oEOCEMcVszAMsVR=26leAxqX6}q%&-&6T+-* zRftQWOR|b2@_j+b*|}o}=H7J|o_hLe43LDCu*fRaRo^%;i>+1t28pC+#P@=;bdamv z$Uc!6)jJTQd--BVb|rS_)u6!C3{!L0%CbpATQiIC_JLEVIiH5#g|=W`zfeq#3u{-T zS(xgoKm1KYT@7}xUyE`HVC4(-J<}T|(yVf@s-cAt-Jw&mn4mtlY?B(akda_cOxh~k;4l#aNFtmNcmy!Ps==-suepCV+8jU!5>=?|F#}yP7;b>MNDjTi1zPADUlTwja+kn%hwJ2{e zqrqxZ!RS{RMJO?v@c6D%sFM)F*F%qpm_CS#j#U#uFg>E0JAy>nnyfgm=@X>x-+_2> z;Jt(#S65f{y8VtjRD`}?(lSQuD)(zsU03)?=y2l12^D0Hl2w8vF-YqQ=jxh~W9-^7 zGN<#=IY`{p+VO8QFW`T+Y(;|3f{7C+;JKu7yfCFdO7e2h?CA?5WTqv~DlwWdJvJD7 zvP!TwryS7|QWObcd$LL~IMfeMTsug7VXGfB$%NEMFxrtTDu|P^Z{tTew0XTMwJce( zM5&2Z!@Q7T!`!)ZRiTl==g?up6p7uTz4*tZH~q{R%#@&P>Ov!gk1{(3w{h=3)D~H*sv|W>t-#Khn#@PrCM6WpW&! zr;t@=G^+4{Wbro?hkZG^Bb2fzF0Oof7_UprAuZ|olH zD1&5*lXheu_KTvJP-{LaCME_GCrwfXK~_n%#F%w^XV%i>`7AS&ST~90yD~>b7I#A! zbFhrT(En0kT*4QfqD(IG8y!$+a!?9rX_1xkN6S`lmOgtqJ19TLV3j$a&ank0we@)O za0W6f>R`4uDK)a23}%Bp^%&C2A0vAQNi5bDPstA&n*bxrIP#u@A|V!2BXZMZRonhK z&i-%&HVH-87Av4EvxJ0%V8zOnaB3^?F*#=6JCdUHyIZmtXC$YjAobKK^`(|nvLjqg zxI03b@Cy%4C zzE1k?qpAh0f?ku5fNQR~25Z-@#oc$`jazQLRrQAx&A(c8w5Moy@NA&4pa5xUX{!Fr zJdldBn9WN1Y8LrBdC#bxxm9IlrHaQ|AKoky_JD6lFNDYUM`YiD2#tw{j|8A@l4_7V zLPA1_Rl-%2Ma6L;AnEpV$qrFDbcan6gPG-Jk!M)U78r$~;+%6hpK@FgRF0@P((C>qhb*d)*IddL2&o-hEDGe zovcnS&Q2=PCSjCOvyk2-YU8E>YfE)im8dEyL}^|&io_34HH%_@lIHW?SL1X0*g2|QV#{H-XgyH&nBbV^oF!5x!7Kl2O%0|V76 zI%x1Ux-rV(!LTF)gDah80hnkjUQ!1_xv8*s;nzFlAxD{JKmUC-#H)->H%MvS+Bu+S2 zIQl4tM)hJX(xtr&s`KY$$dQrfiw%2XrGap=J))} ze2g`2s+S5<{!xTgU*8}BqBlZ9LzNq-5S6+(yQm$aapT6R-&4gL?4O@~R*}GRFgF>P zl1-m6LlMZy;J&G9XEc0+AW@Y*Kcs&0}r5Izy6|{ zWTg%c*jY^W(Z!3EX|fzVYxZnKGAD$q9Z4K9VuXsBSX-xq=Y#}{3N^4V+*c&|F4DK| zVyez?`C1fkfGm^^5;N26pMH$-irNttv(&()n;jdfM4Sk|*Dzd2V$Gb}@B~AHJMO$wZQE11_8{>j zOba6}2UkQEXOH1U$C(L|WS4oC2A*Z00;#mAQ>Uu?X!A2?&QvO7r*FIccKG}IE261V zU0YDhLG=>T=ERH|JsSTuf4(Yb^SRtrW|FUgzfDX`M0QqI+cy%ciRw8aOlLZR3bQxQ zp5ltg;=?*ZnM#M*;!@sE%+CHCL6j(xo0ut*u+x;NBn^y<-0gy5`a_b>wz+}8_uL3& zSP&uRNB^XOYGiSeq>D&Wc3sVQ2~91y#i^z^i@Sz8d&N$zo?as*s-<_+C4C@(NrF3?kfE)-(ix~sf^i(vo}SdzF9$y>-MJDl9@7TZ z9jGjJjb>*L=6hUq8E|$6USIUcBPw}*YxQd7=dKohbu;VnRFka(^{($)0y7JYMVS`jrUQ{vP_L&Kz@B-)pFpe3Ip2W)WINv?u6Ef4ek8&JGW$ zvM$f)n!6#)(+E}&SV>@j$c{4vm@3IMCC`YNr|WpYj6o*v;}U;Rl4%}rB^X3c5>+olP! zRLu80NBZYw%hbtco*A2c!wo9==Yd>aV}*n#r8y2B1m+=N63oLP%rQGdrwI~wK>qsT zi#RUlmFs!Q%L@t$RP@YKfQ*&c!B<~-1rZWu z>m_Vpjfr5y{rZX(YCDP*O%l)3#JrD(YI$Uv6$PFJWfhl$rqe#1nu?J!ASUoUJNn+b zb;>;ZNZ!f*5Ly*v>(;I6`Sf!nn-jo-Aji1;g%{M3a^|2#(kGt&V;{OA>moDcOuzH? z+c+Z+eee%|z$$sqTG8;~!xO};aZAfOgbdLA%&;^z8*Oy<$G9ixs zg=aT8CvsS@yIu0bw^psfU2>gZUPm>vOhZsjjLu(>31ni;oRgIU zs&SnV#hmn&*IvU*^30(_hpKzn291W3Oae$4NqBPUQbb7fze0Xb3*jd6f617rD5ATS z`XaK@)6-SCgUdFh%K%b~lsPk8WjI)#J*T3WywW#NReaH8xNK7~R1QsrWfXRNjS$5) z*m+hX_?{iVLlR?>LnRR$D4{8k$Aa#=|9(|j;C_L&vBLW~xjY3!3!oX(8rT+VVyg5l zO5(G5eaRC~sHC3H;i({|PdpLC-?1*nzIK-$T|`!PPL7J6nep4NI-E_O3t*Z-dnf4A ziY5DJ=RIU8r%!uGmTi*8`uA#43(b(FT>2^ch}UFBzdv$Bl}}j1XUT@jC=hd?i4&|( zaey=pf)xt}s4T(I@6g0}oo&#(=o^Xs2M#FpGZqUJ6J-FbO&BL$i|%>EoER+Q;5F`) zGX>*#Y1&=k;?+<(LD078AIe2N+qZ9517%5;s^ClFW`(Y!KHJjl6aJwonm2+&iMWJ-czQ?d)d~oVZ#&= zB%UQ_9^#FPj#f!C>-s$1Le){VG(DOsq5a1v*4!DC(p>3xcs7GE85OSx7Pmo?ISKTY zkx^0YX%k7-s;3`*@Bu#GxKa67?riZMx}C0YQCXQCp&8JDa`$DJICh?*VaHgkW6&2P z16v^9@qZvt=B$si(8g*2XP+u#rIL=D#PjeN{Tt16;lhQgbM!-;SrSdZv0%XhRWeqk zY9X03T_d3YYhJ9#l30c%G$n=+>|?OJSkLD~%o9^*C4#n~ZWnuwCWV>411mA10zplZ)KYR{Z literal 0 HcmV?d00001 diff --git a/website/static/img/logos/circe.png b/website/static/img/logos/circe.png new file mode 100644 index 0000000000000000000000000000000000000000..23dbe5fc31747675afa62ade5a359971bc46d149 GIT binary patch literal 2428 zcmV-?34`{DP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGqB>(^xB>_oNB=7(L2@y#|K~!i%?V4#+ zR7V!a{Wu?!IB^Hup5=GoG8BvVp-~tHBPP6l$y0>3rD=ncDG<@)$^FO`3>UV!s-MUqE9|J6`l>rvE z5nIMth4gC?;{bE3(A_o;^I&N=t6z&42M{%V7-t>A8f&G5iMgUrWoi-3_tpCNH6q5R zVCjao4zu9m7R|5n@!`K+_ThYQqqlr8e3UnXo9jUIR^4E!7bZGK;rn_F?|_z7f2h9GUS8PeTsk>NEF>5}L@DS7%NY;bhK zM41czFwhmAev5JIr@MTQiH^I2+)E28AW+ z6T+a*OJ!$l?agREegL&GUZ@VVM2*T3HM5<&R2SidmIdypSfa#Q z)59x1NP_*O2)IsL@LYL_kO>I>6UI-T`+QiEnb2^H88EdDMS`mz3IiOV4w0k&qs3^s zeuW*9wLh*v?Y}ppT;+t)K)HBzM)@oklm^+Ldgn(#s~C~iM$|VpqpGGJot>ThyN-?y zG&HE8RyT2Y)zvk0iM}VRsHnomE&D`vn$t6^R;HqthndfSjZFwXndXh#0k&vTIiP-5 zBHGIC@lDBEZx^9z<8qV*+p`F~X3r;@iwxK)06Wpz)~@+Z7y;MRKKTQ5Ej{xb_bR15 zb9s30V6_@!6^fyvr=9JC@lQ_=lm%F$DcT*4hdxDzxTikiIa5)& zd^RfIRYCn;IMypvShwX1ZbxzJJas;{?KsFitF^*w#h4$Ph!FzPc(gD3CPsr`&E+CA zEQ9gpC|?X2tAyV~1&(;TqB_VD>iA%&vyMydL?3pvqw&;XJdE+d15Xo_DQuw*8;_5z z<=FaTdsy)pYuE4Lg=5%AucuZ$gJEgWa1uQc|L$gne%;syMi!xn zm>h^qA4k+Ht)X5UgXU}JCHG>OYpYPVe;djprr@!UDUwx6lsv4Fu4=5ky#ssqCxaHG zAx6{L&$GS_hLxEHz};&Rhu@{kMTm%8g<)a|eM9EYA+prP4;OvMa}TPwtwP(qI}#4i zcK0V#Z&`uH3whEv8p}L=9)TgtFvQpsGLfeA`6U?)OF7ZSeZf=lb?kUHru)X>%@NZv zaEt<0Hlav#^Fpzo4Vvdpf%?nsXs>&^C&?~fxxsNs_cR*qqt~CV41%Tju@E3rMDDB6 zNk3$2bu)JDIm|h8;0R9)GE>6QUWpyk+;QLE8ZArw(Vljg0$KV6@NIH7?8Ihiv`F~+ z{5>m!U`LtwBPMPwzWC}W!?m`yVa>YTI9Kq4fp)Nl=6)AS?TV)Ed0Ov9J+&T(sBcQ&$GT(r_(WoQSK1SA`o*uhQ_pKG=;{BHCeT z?P}po9%9Omf0X$_W*N#cJzRVqDq?fProG&*w6qd76I6O9)Uz@ecH!IWP;madn7x{$ zo|<+V(Xp!;z)Wm@l6HQLMzI=cEhX(N{Da=%NoXb(7D|0XM|mM4Y879n?I!JcUkdEG zf(yvb$-(~5kBA|T7ri`{Q`^yFS$v(0_5puS`bum_0y*v}(MK;_&(~fmY~J_zIB_x+ z+1a_s$;n01j(u=;n~T>5O~s7x_cI-*@4|G zF5wzlzkod@K~k?4-MHCRaUuKu%oT)$y~nV0pWhYsSn>%5zI>%noXwup4ydL@B~Gem zX^Iq@is2hW+@PBC0k?Unu&1;01n^l@RzBj-WL4F5`0M6~2;{?c zyZF4NHL5v8MX!PJ7~fujl6nP8PU%^|NO8K|wdZRwd734c!2hn_;XNxkrMT4ohU!yB zLNt*Ie<+q(gUkFYS0!<;4-;oG-SW^c=@((iDb3!<`D?g{rfW3pzJW`5!q8!ndXp+D zR9W#?#3MC5Uup?7h!nPc*Ejr7g+hXAS^E6-r02qFo%)KC>B*Ecl1reesTqg=lY#mM zHOH(^z!EMg(F7p~N0%t<%bwH?mTpY37af}@R;4poAtr;aQ+iLZRG6cTaP7xi5+b<%wfS!cU}ZP6E5APL z@VBTNENQHjW@7P-7L!ZYsTQ`jwq8Qi7(F&}agTnIU#0eaDZQ{-=^H*REM)?+oZL%L z&Rj0?sV7E>5PG(KU#dq~eIi(T)I+t^_=GLIs_Ev}7b$~bski0!VO*x9g`%(HvL`XD uA^&e+C1hcv!Z>*tzt`91Majlt9{vw7UUw8+!z>a20000jX6_}I=6)Bp%n&w;ntKa#KbX0WC<-x+CP&ALRicED+#WsDX^6OnG9El5Zlpi?5V|4|S-?Vskw#=(Pmr&UCSdvwcJ0V+J=O?e3QGfGPdTGR}Xi4x00b!aXCVa$>qMUig!Iyu8Yh_i9#K!eHu znjJK0NPRexW<}LYF#@T9t25e91+wODGHc}ZB+(ph-R#glCV~X6Q!{K9&XE%;1>G8s zVvCo=Fk>OFKx5+bpc;9!k82xj$`2SPPSREC&|09b_3)6**pwUW&=)^URg)lgw z&jdYL$+JoJo>X`m=Md@OMDr86h!iEhN&en{^<-yc&cI*-neBU)>#Di_0@ko$b2z=b ztKKLn-QC=PE-rpTGokn8L-5ewy@a!2>{uDReLnU`d04+YwB$Eb8?)grEYx866HBjM^AlzYXJmZOC#^%`+_wOENRz!!n^7?W82{`;Qf#$Nya ziPe+GIy=ut!XA*>DbN!7{l-axw0lDtGt)|l8VE-5{8W|xZ2c>Vx&@J($*kE!b{nC_ zk}o%|%~W+S@gz%C#otNwJ7KCs@zoucD{)udXT4L7S{mCA{m7BqOLHJ%L)@QMwf<2f zop;XGF^ezqtC?j$S;fW8YqqW`C9$kY!#3`iw3C~}+hAx#^HIBwBk z0BANnHkB~@FpcP6@YZ{`6?((0c*Z4~Z~!EOA@o-yRT-C<)>OXMj8DV|tIx^wZLHss z5Odv^$2E1~Rk_-WyXoYHtz^9(x}h@rzxTH(T$OPKufxy++jgm+fzN?g*#rZ#Nad|0 zTbrV#s=$IvyooDx+*-S*%W~cJ3-kcOdo?>=aiIpQrfs16BHzcC?LgQo0TbM4Ymet} zeGzrxxe!b)Bsl<)`n%q8gTitZJ$f!}8R<2G3fH0lXTz?Lq zUxbGl4qHBouztWhe~TIAk~j>lcWVpw`6~Lzc#MUjKzEKrkzlhygm3(jGqb8o+XED+ z>B)YHypItqLhRSwsup;H8wvJ|%#P2Iti;xfEOwi3#_9g$_8!sW2|D)Ej0!92L-p$g z+(_AsPB(B&tk`fbsdqrnAVG+jt-oiX7VvCu%@6yG^SRM_NJdprV;eY>`^_H`iK?Tl^(MKOydU0;*UzXc}abH7$PV-nPiS)$_9FcBXfhxlHkJGsZK)XYq|uhH!q(xaK!` z1*$!wdTFhZO8Fwr{Et$&c58Ga93ZtDKOY;|i9HngG|Ffi-R&C{}rHS02C61LLe;=Cgxryrf72%+QiHVi9{ojJ9l}4ocbT&Qe1d+B>DdXw2#3>0swHf LzkqE#OT76H5V0+J literal 0 HcmV?d00001 diff --git a/website/static/img/logos/spark.png b/website/static/img/logos/spark.png new file mode 100644 index 0000000000000000000000000000000000000000..ec8a301ca4f5d1e42c70037bcb097cfaf6868b07 GIT binary patch literal 5548 zcmZ`-Wl$8}*WM*|aY!?CnzW={~ z{U6RS002!_k(bf+G2J)z_J@wco^7{MBn+d{m|n{Oi?K%XB#;v1{!#KFke6?O^3o*a z17tKIOq|(~I`Z2uw-U5u^BPD-P=TA`;x#B+FEY}lf7We_F4hP0)9IgHw7VaS7bnZ; z6+t1SL|QwEeY!qEadR~FxgG3M63E!6Tji{o*a^G*lm2 zox(G6oN@FZ#Wm_N7$LqK7XEO)z}raG)9SiUG;;q5R+5oSK6Dz+^tTM@c<#h5 z>k0MxG#N^ltW)`)wmp=wrp0f&s7%Pe0h8q|7%jOCdj0oIQi zBgRK;Vi4TDxhrdGx~>NZ^6tZ2EF1fLU&A&1W~!@ms#p05{fq(YSpAW{N;HpgrTEiK zIvuo6#B+;Sgw@hnVS5OOzhjhj$8&<;Wk;Ql`lb3H%2Pv=?8PFhpZyCvcjFcw%HW+s zF46fZySZKD4X3rU48bmRb|}7SPi*A#5sTb_v^$0kKH{uB+cXHNQ-m z$$ePb4<}bpaTa1ftC@)mKOj@C$bRg$KY2stR&J_U0b~h;#@P+zaCi^-UCo~-194G{ z{8_61No=aKFWn!NDE@f|ciZq(vGpKNvjwEk5FWP5qx-CuDLP#rw$!N6Pmhb2 zhRIHDQKYAY<`cR;$1Lb-^GE9Tc-VF7SGR)w({-xp7CiXhxcgmQru5T`bs_ua}Szvp3e@xrfjB4JqIK;y)5>Oei*V8G$H!l4v1|LcC zjM+ryuB6UraS41s6@MA9Ox!vXuQH!CY^t>q1onD@b#n0Pu#MHFT) z#bK#`9eL~BR%or(UeJod*wqja84yJava@HGm@8I{?=DvIB_M|r4N{g#DHY!lOPUq% z1>Kj*F8PyOi=R?NC3u?CtIpJWigTOYahFFKmIrhl3n8PQT_0Iqf0R-&aGGd#+7OCxkou}l%7`g-p^TW1)TCYJP)#uOi=H_UfQmKgW z=W_cpW|V>GZ*$n~Iwz^A?r^4R!I#mUKe507UM*LYi{RIFE;>Vb(V-%m(SN2bnJ1c;aD zms69k&2o9}rk1}xx3F0D*dAg1G|<~if;^O}lu}QB7S3cwt9Ew>8&FqQ&rKfFz25yM zS?gv^K=(d2zv-Oo3%!UpM&jz~N;`=yb{iwq;84TGL+k>fe5P%Ws}zKlg(m<_+K_kx z!NM;kkeKyg9n!yWU>^8v*v?fY02`2Qgaj{4LOoO%{>j7A&JtEz?P0wRk48IHKId8LyOGr+xX%eDKRh} zO-)V&BI|UM`yewlwMVrtZ=#$_VD#tDaKd4gitMSCoY$&r!`}lJ->J5xK7TCe(C_U7 zLX6MPXFM_aj<6(m{!%H&B$X3nLAGWzKdTiiNm7`D#rgIWs);tx$1%5QlT>t#tkyfj zEhK(%a42QZj7lyh`F=?YyZkckZnwA)z$EX3Q%1oPK?1Z^_qC*^6ZMzBNg=$sV91)9 z60skt! z=)5j|eoN(wm{QFmsjXnC&^Ae_rkTK^N3fPHJSMWBGyBDM(pT@%oC{c071Vywlp*Vv zsZ+BZGl<4|BGo1DW*6_tDCJBYISJ-V!-iY+oL${cHDEVZ$~Un~(-`)-T&V^vhu7I1 zm`S6qo-Sy;tC_7|Zv27W_?}!}s0?#}I$*Q4jaFymy1Uvm8w>1^6Y#&}g%A@bb(h~4 zc98#Z2XDyPDG-tHN1pS)4p&hFB<|k z$_q!!1nOr>cb$OxH{r@7I+jJ)gQov1(#L!vE$FAIv0qgv_pCj-PkA?Om44eiy-nW-_7~1C5u@ZN*jrP4l(DI z9?F#5iskrbyr>3d3IWq-kj4A=w|gX}pu`&^CKD@hvtTm766`!C$4(swVm0(U8NGpk! z5`wH2&vU`wr{p2gG_0m-`RX)6Mk&BB1N??NNxEMjTYjhW$4g$~kr$d4jQu1-Y!5Cc z3%Ic42TG-##5tx_=EG7H3^ZO~Fys$^Tyr5#;iRQx&?Viu4%GATZ>&+hqYO5fDC>Qa zX?8*a31qeMWMp705Hqwv5yZ)R#Ay;6If%?XiN(E3W= zZTsWayNqR3f}$`y^DdF0teiBW_V^q1I>bDlRqEY0N7wDjf#>;bySN!9KhHeRYH|tS z`R-|dW>PE}DgGVbFz5h!=-=l$YE;^k8Le~m`5kqIZvA-H%IcejcC>l81m`?Yhd|P% z{a)sJmsuHPdwe$f`Xbk&3k^wk$MKm^v~eu8oKQ|7{!C~-7#s?Ul&vsrweOYT)sP1P zqWry=ZxY2l)@qK)QM{r>qakvn{PO!As=f$rKEJGBYndf;K6XvEH~g?HG0eq9q0L~r z9sfG^c{lt0tLI;z+℘<~hXkF{~7*M7|8zoi+Y|M#s991bP3U4!nGqU&*1TWy9!C zC*h-PpDg4wJHLKHiAT@(YYIp#JSxiAPZBr_7 zccLFxT(uf4;+T{qjwcqWugRm{x_PF_3bV_M}yP``uboE2EpyoemNR+v)4s%o1sj(wT`H=od4lOX?l`yah);O9bT;y%e(#{zV^{Z0{~p#O7Qp&}LK}qwn9`S;@vF z?6=`zNCp+OpYwT257@OR2j55_=y2KDz`bN7w;wlusZNaoHKj$y>?-5qiCalTBk&c} zb~p7q;G*BUCIm8f30w2Cv0$Mu-)#+BIQ3v59y^zDVy8wkl?WQlZd_;ko!bj|?ctAo zATLypjE9Jv%Y!P)}<{o>)H&)y}|z2U6RnE-V!{N;*TYC}_Op{(z~b9s3k$*XH^&C9KrgZpm&HXU9T{5t;jBGPS|vaJ5$DS@o5ZLA=V z0Y@<<7h$eNZey#eHpJTaviw-s*hO+^@x7Q88*S&SC|7>WPYRa$&%lG$2o7$5@It90 z1UTR9os0KJcoJJ#7H^VROd6+>zU(wemg9J(m45GM2;P`YKZvxN(+UAlSChVg(&7Dx z`$*TTVyMCp^z_Nijsx;lo8EV4schp0@Rnh{hm!syA&@X=u+`^pwKs6dbhsx9P)_0P zoi5GjBTPi~evh@}C_%zlz@;L#A5DzxN^$_v^^#?prf#;EIQjwc3~ltmTc#!05Fv^m z>7Ou6;009^1GscM`g)|Dp2~TWSe6xEP1aQV0I!G%q(^=e&IRi+1N-(r8lQdy~OLdy)@4FU($S#fNN&3bhV zc51-mO3V8&wZ;=PrQw|RUN=a))vumxAqb6bez~90jRYNjwE&yP|ENR8N%t9v?D$mN3|8N>*u}?>Ou-Z0V^9U6E^1C!252zJLY)JJ(*)Mu zKmYrAVi46F;tRY*J^fv_3YlrXH$@ven23l@K*tD>dVaxSY0Jly!xoMXS8E?2pylGF zNaG~gfwkUeLqo-eY4^rU(I%nqBEqTC<9vkNjg@eB*lyLsw8lr_&+!=csql=^4e>=q@p1!$ zFx`&bC0?9N;$)GaUFchQ6~$nD7yg{<2tH= z#;R|r{tDWwzoyAM$-_^V;GsXM2c@g8_ZzrRo+7b%p4p%AyM1Kj=Z_xYkCEgcuN2sm zt+;th_wRPpPqfZLAB34;W;Uxt4r^(~(290eXh^^77&x?=iCvYtK3#5fbxB--+pp;3 z8%Psd+bqqGuRL~$h@i%|(n8Ue0D#)TJ?lK_LJtiKJy{M>rr{1;=kl zIE6P3!ooy14;l+VAcZ15I%4Ts^O%?NhasQNcDnh5i-#rT>2P%3O5(UZ>lj!;BoIly zm7^>v1ddPb(CN**f8TRMqCoriGJ=q1ggO4%>jXNSyLAe+vc`k6ZT?z^O=nSPBsZ`& z#I4#%yxK-&H8_HZMpiT4uViip$d&KDd1Kd8Ml}{&>Pm-aQ1WxK)J!uDQ|mA}VS!7~ zK{ov5ip!n`8ngiDhp(yAZq4NQps>cu-m;8}YC8>Yqstwgi3#UN6&=CmKbl zs@(f|_0YQ6=)jfLRlAFdMONA%rlzA6#K^E$4~N`d&ZqLg!JWD@Ov}|w?}%ZBAgVy% zD2`rSQAt!Du511k7fMy`U55QrwB&98du|~ylr%3&Nk>ViLa!MnSY-B9b{eG(Wdch- z4Js{-t`x+hHm=iOwUCoP9V;`R1BWDass>ldJ^|7~{xWhSqZmIhAFi1G6HcHL`jzEV z{MF|P`m{Vcx`tBKP_)P5hK>h!vTH>v$eAM!pcrAV_bclMw%x6o_Dm5ZC0tI0nK&q* zBV^zO$iwd7O2l44H+uzPJVOp>QMML1LO`|jzQ3JImX+QdqsDprbWHb^LAKj>j_|eh p4Y;uC;Ufkgu#kU=YcAt8Lv6i~qx|3}`G>hCKt(}AzCzYK Date: Sat, 23 Jul 2022 16:02:36 +0200 Subject: [PATCH 06/23] fix images in intro.md --- docs/intro.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/intro.md b/docs/intro.md index fbccb59e..68934679 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -33,12 +33,12 @@ or a BigQuery table into a Cassandra table without having code that relates thos ### Available conversions: -| From / To |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | -|--------------|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| -| Scala Types | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | -| BigQuery | | - |:white_check_mark:|:white_check_mark:| | -| Spark | |:white_check_mark:| - |:white_check_mark:| | -| Cassandra | |:white_check_mark:|:white_check_mark:| - | | -| Circe (JSON) | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| From / To | |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | +|:---------------:|:---------------------------------------------------------------------------------------------------------------------:|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| +| Scala | | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| BigQuery | | | - |:white_check_mark:|:white_check_mark:| | +| Spark | | |:white_check_mark:| - |:white_check_mark:| | +| Cassandra | | |:white_check_mark:|:white_check_mark:| - | | +| Circe (JSON) | | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | From af7a1c47004061651ff0205441acf574b40fad08 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Sat, 30 Jul 2022 09:26:31 +0200 Subject: [PATCH 07/23] Update google-cloud-bigquery to 2.14.1 (#234) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9a0b2185..6e708baa 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.13.8", + "com.google.cloud" % "google-cloud-bigquery" % "2.14.1", scalatest % "it,test" ) From 44fdadb9026825aa4304e656ac5b89a5cc117ecd Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Sat, 30 Jul 2022 09:26:45 +0200 Subject: [PATCH 08/23] Update sbt-scoverage to 2.0.1 (#235) --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index e6427e45..d37b4a7c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.13") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") From c6e73ff65a67d9cf3c50f3e5d2bcedee9e3561f5 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Sun, 7 Aug 2022 15:31:41 +0200 Subject: [PATCH 09/23] Update google-cloud-bigquery to 2.14.3 (#237) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 6e708baa..e30045a2 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.14.1", + "com.google.cloud" % "google-cloud-bigquery" % "2.14.3", scalatest % "it,test" ) From 8cd92bb390eb4e9652865e08b0f643ae7419007c Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:28:21 +0200 Subject: [PATCH 10/23] Update sbt-scoverage to 2.0.2 (#239) --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index d37b4a7c..bd73aa12 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.2") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.13") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") From 5fdbc7eb7766dc3ce15e646b9e1a4f98d75683d8 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Mon, 22 Aug 2022 10:15:21 +0200 Subject: [PATCH 11/23] Update google-cloud-bigquery to 2.14.6 (#241) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index e30045a2..df3953e0 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.14.3", + "com.google.cloud" % "google-cloud-bigquery" % "2.14.6", scalatest % "it,test" ) From dd6930d1f990e470ce3ba6ca4b18b0ca4184ecf8 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Mon, 29 Aug 2022 22:04:48 +0200 Subject: [PATCH 12/23] Update google-cloud-bigquery to 2.14.7 (#242) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index df3953e0..3c240aca 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.14.6", + "com.google.cloud" % "google-cloud-bigquery" % "2.14.7", scalatest % "it,test" ) From 33329de5fe65392dd8b4560392ecc72355194457 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 30 Aug 2022 19:25:55 +0200 Subject: [PATCH 13/23] Update google-cloud-bigquery to 2.15.0 (#244) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3c240aca..6c220b60 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.14.7", + "com.google.cloud" % "google-cloud-bigquery" % "2.15.0", scalatest % "it,test" ) From c5d20f261e1a81ab868cb0932fa93087604f0e53 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 30 Aug 2022 19:26:11 +0200 Subject: [PATCH 14/23] Update logback-classic to 1.4.0 (#243) --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 6c220b60..f2efc341 100644 --- a/build.sbt +++ b/build.sbt @@ -39,14 +39,14 @@ lazy val scalacCommon = Seq("-Xsource:3") //Dependencies lazy val coreDependencies2 = Seq( - "ch.qos.logback" % "logback-classic" % "1.2.11", + "ch.qos.logback" % "logback-classic" % "1.4.0", "org.clapper" %% "grizzled-slf4j" % "1.3.4", "com.chuusai" %% "shapeless" % "2.3.9", scalatest % Test ) lazy val coreDependencies3 = Seq( - "ch.qos.logback" % "logback-classic" % "1.2.11", + "ch.qos.logback" % "logback-classic" % "1.4.0", "org.clapper" % "grizzled-slf4j_2.13" % "1.3.4", scalatest % Test ) From 1dac1deed42e9afa5de79e0ec11aa4d763257896 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Thu, 1 Sep 2022 08:05:34 +0200 Subject: [PATCH 15/23] Update scalafmt-core to 3.5.9 (#240) --- .scalafmt.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 269221cb..c3c6c1b6 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,5 +1,5 @@ runner.dialect = scala213source3 -version = "3.5.8" +version = "3.5.9" maxColumn = 120 docstrings.blankFirstLine = no docstrings.style = SpaceAsterisk From 4a6ef833c68a2cfc1abfc7bd97c912646283342e Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Sun, 4 Sep 2022 09:15:55 +0200 Subject: [PATCH 16/23] Updated Scala to 3.1, Circe and ScalaTest --- CHANGELOG.md | 6 ++++++ build.sbt | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cf37eec..6a25b00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### Big Data Types v1.2.1 +Some dependencies (Circe - ScalaTest) have updated Scala to 3.1.X +- Dependencies updated to newer versions +- Scala version changed to 3.1.X + - This version is no longer compatible with Scala 3.0.X + ### Big Data Types v1.2.0 - New module for Circe (JSON) - Conversion from Circe to other types diff --git a/build.sbt b/build.sbt index f2efc341..53fd461f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,12 +1,12 @@ //used to build Sonatype releases -lazy val versionNumber = "1.2.0" +lazy val versionNumber = "1.2.1" lazy val projectName = "big-data-types" version := versionNumber name := projectName lazy val scala213 = "2.13.7" lazy val scala212 = "2.12.15" -lazy val scala3 = "3.0.2" +lazy val scala3 = "3.1.0" lazy val supportedScalaVersions = List(scala3, scala213, scala212) scalaVersion := scala213 @@ -69,7 +69,7 @@ lazy val cassandraDependencies = Seq( scalatest % Test ) -val circeVersion = "0.14.1" +val circeVersion = "0.14.2" lazy val jsonCirceDependencies = Seq( "io.circe" %% "circe-core", @@ -77,7 +77,7 @@ lazy val jsonCirceDependencies = Seq( "io.circe" %% "circe-parser" ).map(_ % circeVersion) -lazy val scalatest = "org.scalatest" %% "scalatest" % "3.2.11" +lazy val scalatest = "org.scalatest" %% "scalatest" % "3.2.13" //Project settings lazy val root = (project in file(".")) From 62e13b281c57205a39f724039ebe834ebc43e986 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Fri, 16 Sep 2022 21:20:34 +0200 Subject: [PATCH 17/23] Update logback-classic to 1.4.1 (#248) --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 53fd461f..695f5a30 100644 --- a/build.sbt +++ b/build.sbt @@ -39,14 +39,14 @@ lazy val scalacCommon = Seq("-Xsource:3") //Dependencies lazy val coreDependencies2 = Seq( - "ch.qos.logback" % "logback-classic" % "1.4.0", + "ch.qos.logback" % "logback-classic" % "1.4.1", "org.clapper" %% "grizzled-slf4j" % "1.3.4", "com.chuusai" %% "shapeless" % "2.3.9", scalatest % Test ) lazy val coreDependencies3 = Seq( - "ch.qos.logback" % "logback-classic" % "1.4.0", + "ch.qos.logback" % "logback-classic" % "1.4.1", "org.clapper" % "grizzled-slf4j_2.13" % "1.3.4", scalatest % Test ) From 9b1c27bc729d691ed8dedfbb0d5cf5fd5c21311a Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Fri, 16 Sep 2022 21:20:49 +0200 Subject: [PATCH 18/23] Update google-cloud-bigquery to 2.16.0 (#247) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 695f5a30..532be148 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.15.0", + "com.google.cloud" % "google-cloud-bigquery" % "2.16.0", scalatest % "it,test" ) From 909c39afe7400b260de5a176b736f0adff43d1de Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Fri, 16 Sep 2022 21:21:06 +0200 Subject: [PATCH 19/23] Update shapeless to 2.3.10 (#246) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 532be148..708137ee 100644 --- a/build.sbt +++ b/build.sbt @@ -41,7 +41,7 @@ lazy val scalacCommon = Seq("-Xsource:3") lazy val coreDependencies2 = Seq( "ch.qos.logback" % "logback-classic" % "1.4.1", "org.clapper" %% "grizzled-slf4j" % "1.3.4", - "com.chuusai" %% "shapeless" % "2.3.9", + "com.chuusai" %% "shapeless" % "2.3.10", scalatest % Test ) From 4f94b5f73c7e1e77fad94f7275a104d388bd2773 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 20 Sep 2022 10:34:20 +0200 Subject: [PATCH 20/23] Update sbt-scoverage to 2.0.3 (#253) --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index bd73aa12..0833a6e0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.2") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.3") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.13") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") From 77a9dbdc678f260f80a53ff0dc1bc2a6b602fc36 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 20 Sep 2022 10:34:53 +0200 Subject: [PATCH 21/23] Update circe-core, circe-generic, ... to 0.14.3 (#251) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 708137ee..a69cad2d 100644 --- a/build.sbt +++ b/build.sbt @@ -69,7 +69,7 @@ lazy val cassandraDependencies = Seq( scalatest % Test ) -val circeVersion = "0.14.2" +val circeVersion = "0.14.3" lazy val jsonCirceDependencies = Seq( "io.circe" %% "circe-core", From 2f530e75fa5d86c665a4f82b0b626d015e1c6843 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 20 Sep 2022 10:35:04 +0200 Subject: [PATCH 22/23] Update google-cloud-bigquery to 2.16.1 (#250) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a69cad2d..12ad843d 100644 --- a/build.sbt +++ b/build.sbt @@ -53,7 +53,7 @@ lazy val coreDependencies3 = Seq( lazy val bigqueryDependencies = Seq( "com.google.auto.value" % "auto-value-annotations" % "1.9", // needed for an incompatibility between BQ & Scala3 - "com.google.cloud" % "google-cloud-bigquery" % "2.16.0", + "com.google.cloud" % "google-cloud-bigquery" % "2.16.1", scalatest % "it,test" ) From 617371ff8f8518de24bb769e06a0d2f1c07d6905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Mont=C3=B3n?= Date: Fri, 23 Sep 2022 23:11:25 +0200 Subject: [PATCH 23/23] BigQuerySchemas API (#249) * Added BigQuerySchemas as a public API to return only Schemas from BigQuery, without creating tables in a real environment. Changed JavaConverters to accept Seqs * Seq in Java converters for Scala 2.13+ * BigQuerySchemas with `schema(value: A)` being A any Product type. * CHANGELOG.md and version * removing unused code. Test renamed to Spec --- CHANGELOG.md | 8 +-- .../bigquery/BigQueryDefinitions.scala | 2 + .../bigquery/BigQuerySchemas.scala | 32 +++++++++ .../bigdatatypes/bigquery/BigQueryTable.scala | 4 ++ .../bigquery/JavaConverters.scala | 2 +- .../bigquery/JavaConverters.scala | 2 +- .../bigquery/BigQuerySchemasSpec.scala | 71 +++++++++++++++++++ build.sbt | 2 +- 8 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemas.scala create mode 100644 bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemasSpec.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a25b00c..2d5cacff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ -### Big Data Types v1.2.1 -Some dependencies (Circe - ScalaTest) have updated Scala to 3.1.X -- Dependencies updated to newer versions -- Scala version changed to 3.1.X +### Big Data Types v1.3.0 +- Added BigQuerySchemas as an interface to create Schemas without creating tables +- Updated Scala to Scala 3.1.X - This version is no longer compatible with Scala 3.0.X + - The Scala version has been upgraded due to some dependencies (Circe - ScalaTest) that have updated Scala to 3.1.X ### Big Data Types v1.2.0 - New module for Circe (JSON) diff --git a/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitions.scala b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitions.scala index 9c222f17..d34da39f 100644 --- a/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitions.scala +++ b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitions.scala @@ -2,6 +2,8 @@ package org.datatools.bigdatatypes.bigquery import com.google.cloud.bigquery.{Schema, StandardTableDefinition, TimePartitioning} import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava +import org.datatools.bigdatatypes.conversions.SqlInstanceConversion +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats private[bigquery] object BigQueryDefinitions { diff --git a/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemas.scala b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemas.scala new file mode 100644 index 00000000..90c2f686 --- /dev/null +++ b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemas.scala @@ -0,0 +1,32 @@ +package org.datatools.bigdatatypes.bigquery + +import com.google.cloud.bigquery.Schema +import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava + +/** + * Public API for generating BigQuery Schemas. + * Any type implementing [[SqlTypeToBigQuery]] or [[SqlInstanceToBigQuery]] can be converted into a BigQuery [[Schema]] + * If multiple types are given, the resulting schema will be the concatenation of them. + */ +object BigQuerySchemas { + + /** + * Given any type that implements [[SqlTypeToBigQuery]] returns the BigQuery Schema for that type + * @tparam A is any type implementing [[SqlTypeToBigQuery]] + * @return [[Schema]] ready to be used in BigQuery + */ + def schema[A: SqlTypeToBigQuery]: Schema = BigQueryDefinitions.generateSchema[A] + def schema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery]: Schema = BigQueryDefinitions.generateSchema[A, B] + def schema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery]: Schema = BigQueryDefinitions.generateSchema[A, B, C] + def schema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery]: Schema = BigQueryDefinitions.generateSchema[A, B, C, D] + def schema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery, E: SqlTypeToBigQuery]: Schema = BigQueryDefinitions.generateSchema[A, B, C, D, E] + + /** + * Given an instance of a Product, extracts the BQ [[Schema]] from its type + * @param value an instance of any Product + * @tparam A is any Product type + * @return [[Schema]] with the same structure as the given input + */ + def schema[A <: Product](value: A)(implicit a: SqlTypeToBigQuery[A]): Schema = + Schema.of(toJava(SqlTypeToBigQuery[A].bigQueryFields)) +} diff --git a/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTable.scala b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTable.scala index ac02acdb..2ffd7b16 100644 --- a/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTable.scala +++ b/bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTable.scala @@ -14,6 +14,10 @@ import org.datatools.bigdatatypes.bigquery.BigQueryDefinitions.{generateSchema, import scala.util.{Failure, Try} +/** + * Methods in this object are creating real tables into a BigQuery environment. + * If only the schema of the table is desired, please use [[BigQuerySchemas]] + */ object BigQueryTable { lazy val service: BigQuery = BigQueryOptions.getDefaultInstance.getService diff --git a/bigquery/src/main/scala_2.13+/org/datatools/bigdatatypes/bigquery/JavaConverters.scala b/bigquery/src/main/scala_2.13+/org/datatools/bigdatatypes/bigquery/JavaConverters.scala index 332fc1d4..68e04f9b 100644 --- a/bigquery/src/main/scala_2.13+/org/datatools/bigdatatypes/bigquery/JavaConverters.scala +++ b/bigquery/src/main/scala_2.13+/org/datatools/bigdatatypes/bigquery/JavaConverters.scala @@ -5,6 +5,6 @@ import scala.jdk.CollectionConverters.{IterableHasAsJava, IterableHasAsScala} object JavaConverters { - def toJava[A](value: List[A]): lang.Iterable[A] = value.asJava + def toJava[A](value: Seq[A]): lang.Iterable[A] = value.asJava def toScala[A](value: lang.Iterable[A]): List[A] = value.asScala.toList } diff --git a/bigquery/src/main/scala_2.13-/org/datatools/bigdatatypes/bigquery/JavaConverters.scala b/bigquery/src/main/scala_2.13-/org/datatools/bigdatatypes/bigquery/JavaConverters.scala index 02f6f930..6ff08690 100644 --- a/bigquery/src/main/scala_2.13-/org/datatools/bigdatatypes/bigquery/JavaConverters.scala +++ b/bigquery/src/main/scala_2.13-/org/datatools/bigdatatypes/bigquery/JavaConverters.scala @@ -5,6 +5,6 @@ import scala.collection.JavaConverters.{asJavaIterableConverter, iterableAsScala object JavaConverters { - def toJava[A](value: List[A]): lang.Iterable[A] = value.asJava + def toJava[A](value: Seq[A]): lang.Iterable[A] = value.asJava def toScala[A](value: lang.Iterable[A]): List[A] = value.asScala.toList } diff --git a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemasSpec.scala b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemasSpec.scala new file mode 100644 index 00000000..197a9386 --- /dev/null +++ b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQuerySchemasSpec.scala @@ -0,0 +1,71 @@ +package org.datatools.bigdatatypes.bigquery + +import com.google.cloud.bigquery.Field.Mode +import com.google.cloud.bigquery.{Field, Schema, StandardSQLTypeName} +import org.datatools.bigdatatypes.TestTypes.ListOfStruct +import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava +import org.datatools.bigdatatypes.{BigQueryTestTypes, UnitSpec} +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats +import org.datatools.bigdatatypes.conversions.SqlTypeConversion.* + +class BigQuerySchemasSpec extends UnitSpec { + + val elements1: Seq[Field] = List( + Field.newBuilder("a", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + val elements2: Seq[Field] = List( + Field.newBuilder("a", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("b", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + val elements3: Seq[Field] = List( + Field.newBuilder("a", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("b", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("c", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + val elements4: Seq[Field] = List( + Field.newBuilder("a", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("b", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("c", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("d", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + val elements5: Seq[Field] = List( + Field.newBuilder("a", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("b", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("c", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("d", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build(), + Field.newBuilder("e", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + case class Simple1(a: Int) + case class Simple2(b: Int) + case class Simple3(c: Int) + case class Simple4(d: Int) + case class Simple5(e: Int) + + behavior of "BigQuerySchemas" + + "Case class with Struct List" should "be converted into BQ Schema" in { + val fields: Schema = BigQuerySchemas.schema[ListOfStruct] + fields shouldBe Schema.of(toJava(BigQueryTestTypes.basicNestedWithList)) + } + + "2 classes" should "be converted into a BQ Schema" in { + BigQuerySchemas.schema[Simple1, Simple2] shouldBe Schema.of(toJava(elements2)) + } + + "3 classes" should "be converted into a BQ Schema" in { + BigQuerySchemas.schema[Simple1, Simple2, Simple3] shouldBe Schema.of(toJava(elements3)) + } + + "4 classes" should "be converted into a BQ Schema" in { + BigQuerySchemas.schema[Simple1, Simple2, Simple3, Simple4] shouldBe Schema.of(toJava(elements4)) + } + + "5 classes" should "be converted into a BQ Schema" in { + BigQuerySchemas.schema[Simple1, Simple2, Simple3, Simple4, Simple5] shouldBe Schema.of(toJava(elements5)) + } + + "An instance" should "be converted into a BQ Schema" in { + val s = Simple1(1) + BigQuerySchemas.schema[Simple1](s) shouldBe Schema.of(toJava(elements1)) + } +} diff --git a/build.sbt b/build.sbt index 12ad843d..189dc930 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,5 @@ //used to build Sonatype releases -lazy val versionNumber = "1.2.1" +lazy val versionNumber = "1.3.0" lazy val projectName = "big-data-types" version := versionNumber name := projectName