99
1010
1111class SparkTransformation (Transformation ):
12+ """
13+ SparkTransformation can be used to define a transformation using a Spark UDF or SQL query.
14+ The current spark session will be used or a new one will be created if not available.
15+ E.g.:
16+ spark_transformation = SparkTransformation(
17+ mode=TransformationMode.SPARK,
18+ udf=remove_extra_spaces,
19+ udf_string="remove extra spaces",
20+ )
21+ OR
22+ spark_transformation = Transformation(
23+ mode=TransformationMode.SPARK_SQL,
24+ udf=remove_extra_spaces_sql,
25+ udf_string="remove extra spaces sql",
26+ )
27+ OR
28+ @transformation(mode=TransformationMode.SPARK)
29+ def remove_extra_spaces_udf(df: pd.DataFrame) -> pd.DataFrame:
30+ return df.assign(name=df['name'].str.replace('\s+', ' '))
31+ """
1232 def __new__ (
1333 cls ,
1434 mode : Union [TransformationMode , str ],
@@ -22,6 +42,18 @@ def __new__(
2242 * args ,
2343 ** kwargs ,
2444 ) -> "SparkTransformation" :
45+ """
46+ Creates a SparkTransformation
47+ Args:
48+ mode: (required) The mode of the transformation. Choose one from TransformationMode.SPARK or TransformationMode.SPARK_SQL.
49+ udf: (required) The user-defined transformation function.
50+ udf_string: (required) The string representation of the udf. The dill get source doesn't
51+ spark_config: (optional) The spark configuration to use for the transformation.
52+ name: (optional) The name of the transformation.
53+ tags: (optional) Metadata tags for the transformation.
54+ description: (optional) A description of the transformation.
55+ owner: (optional) The owner of the transformation.
56+ """
2557 instance = super (SparkTransformation , cls ).__new__ (
2658 cls ,
2759 mode = mode ,
0 commit comments