后台手工启动Defensor插件

  其他常见问题
内容纲要

背景关系

Defensor中执行部分Inceptor的分类分级任务、静态脱敏、去标识化任务,Inceptor都需要加载定制的UDF后才能运行。但是页面开启插件只是临时的,重启pod就会导致插件失效,为了持久化可以使用,以及如果页面开启插件时出现错误无法使用插件的情况。需要后台手动配置持久化udf文件。

操作步骤

1 将defensor的data-protection镜像中的/usr/lib/protection-web/plugins/defensor-inceptor-1.0.tar.gz拷出容器,该压缩包中包含插件的jar包,或者去manager页面的/var/lib/transwarp-manager/master/content/meta/plugins/下获取jar包

注:如果存在argo的情况包名会有变化但其实内容是相同的上传产品包的时候做了一个hack,会把一个jar包重命名2次。复制2个,一个给quark,一个给3.X的argodb叫argodbcomputing

file

2 获取到defensor-inceptor-1.0.tar.gz之后,执行

tar zxvf defensor-inceptor-1.0.tar.gz

将tgz包解压,解压后的defensor/inceptor/lib目录中protection-inceptor-udf-1.8.jar即为Defensor的插件包。

3 上传到HDFS

通过客户端,执行hdfs命令:

hdfs dfs -put ./inceptor/lib/protection-inceptor-udf-1.8.jar /tmp/

假设将/tmp/protection-inceptor-udf-1.8.jar上传到了Inceptor所依赖的HDFS的/tmp路径下。hive用户需要有该jar包在HDFS上的读取权限(注意:生产环境请勿放在/tmp下)

4 上传之后需要执行对应的sql语句,根据denfensor的版本不通需要执行的sql语句也不相同

根据Defensor的版本进行添加,比如安装的Defensor版本为3.2.0,则需要添加3.0.0 – 3.1.0版本所有的UDF(因为3.2.0版本没有新增UDF):
Defensor-3.0.0:

create permanent function keep_back as 'io.udf.KeepBack' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function keep_from as 'io.udf.KeepFrom' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function keep_from_sl as 'io.udf.KeepFromSL' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function keep_front as 'io.udf.KeepFront' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_back as 'io.udf.MaskBack' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_back_sl as 'io.udf.MaskBackSL' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_from as 'io.udf.MaskFrom' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_from_sl as 'io.udf.MaskFromSL' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_front as 'io.udf.MaskFront' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_front_sl as 'io.udf.MaskFrontSL' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_specify as 'io.udf.MaskSpecify' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function number_floor as 'io.udf.NumberFloor' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function phone_confusion as 'io.udf.PhoneConfusion' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function sm3 as 'io.udf.SM3' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function sm4_decrypt as 'io.udf.SM4Decrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function sm4_encrypt as 'io.udf.SM4Encrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function sequence as 'io.udf.Sequence' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function triple_des_decrypt as 'io.udf.TripleDESDecrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function triple_des_encrypt as 'io.udf.TripleDESEncrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function regular as 'io.udf.Regular' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

create permanent function match_bank_card as 'io.udf.MatchBankCard' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function match_birthday as 'io.udf.MatchBirthday' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function match_citizen_id as 'io.udf.MatchCitizenId' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function match_credit_code as 'io.udf.MatchCreditCode' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function match_url as 'io.udf.MatchUrl' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function vin as 'io.udf.VIN' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function imsi as 'io.udf.IMSI' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function imei as 'io.udf.IMEI' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

create permanent function block_back as 'io.udf.BlockBack' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function block_front as 'io.udf.BlockFront' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function blur as 'io.udf.Blur' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function character_scrambling as 'io.udf.CharacterScrambling' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function cover as 'io.udf.Cover' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function cover_sl as 'io.udf.CoverSL' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function des_decrypt as 'io.udf.DESDecrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function des_encrypt as 'io.udf.DESEncrypt' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function date_floor as 'io.udf.DateFloor' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function enumeration as 'io.udf.Enumeration' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

create permanent function random as 'io.udf.GenericUDFRandom' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';  //注意 系统自带了random  如果不用需要把这行注释掉
create permanent function random_str as 'io.udf.GetRandomString' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function hmac as 'io.udf.HMAC' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function hide as 'io.udf.Hide' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

Defensor-3.1.0:

create permanent function replace_max as 'io.udf.ReplaceMax' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_min as 'io.udf.ReplaceMin' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_max_and_min as 'io.udf.ReplaceMaxAndMin' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';```bash
create permanent function replace_max as 'io.udf.ReplaceMax' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_min as 'io.udf.ReplaceMin' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_max_and_min as 'io.udf.ReplaceMaxAndMin' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

Defensor-3.4.0:

create permanent function mask_firstname as 'io.udf.MaskFirstname' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_lastname as 'io.udf.MaskLastname' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_birthday as 'io.udf.ReplaceBirthday' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_code as 'io.udf.ReplaceCode' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_email as 'io.udf.ReplaceEmail' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_phone as 'io.udf.ReplacePhone' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_firstname_fixed as 'io.udf.MaskFirstnameFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function mask_lastname_fixed as 'io.udf.MaskLastnameFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_birthday_fixed as 'io.udf.ReplaceBirthdayFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_code_fixed as 'io.udf.ReplaceCodeFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_email_fixed as 'io.udf.ReplaceEmailFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create permanent function replace_phone_fixed as 'io.udf.ReplacePhoneFixed' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

create TEMPORARY function sm3_hex as 'io.udf.SM3Hex' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create TEMPORARY function mask_name as 'io.udf.MaskName' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';
create TEMPORARY function mask_credential as 'io.udf.MaskCredential' using jar 'hdfs:///tmp/protection-inceptor-udf-1.8.jar';

验证插件

执行下面的sql,能出结果就说明插件的udf是成功导入的:

select random_str(5) from system.dual

file

这篇文章对您有帮助吗?

平均评分 0 / 5. 次数: 0

尚无评价,您可以第一个评哦!

非常抱歉,这篇文章对您没有帮助.

烦请您告诉我们您的建议与意见,以便我们改进,谢谢您。