udf

udf/udf_python.py

@@ -333,4 +333,12 @@ class clean_text(object):
         if input_text is None:
             return ""
         cleaned_text = self.re.sub(r"[^\u4e00-\u9fa5a-zA-Z0-9]", "", input_text)
-        return "" if cleaned_text is None else cleaned_text
+        return "" if cleaned_text is None else cleaned_text
+
+@annotate("string->string")
+class deduplication4list(object):
+    def evaluate(self, input_text):
+        if input_text is None:
+            return None
+        result = list(set(input_text.split(",")))
+        return ",".join(result)