# -*- coding: utf-8 -*-

"""

Created on Wed Jul 11 10:48:27 2018

@author: XiaSiYang

"""

import numpy as np,math

from math import log

import pandas as pd,matplotlib,matplotlib.pyplot as plt

from patsy import dmatrices

from patsy import dmatrix

import statsmodels.api as sm

from sklearn.linear_model import LogisticRegression

from statsmodels.stats.outliers_influence import variance_inflation_factor

from sklearn.utils.multiclass import type_of_target

from sklearn.preprocessing import StandardScaler

import statsmodels.formula.api as smf

from copy import copy

import pickle,xgboost as xgb

from xgboost.sklearn import XGBClassifier

from sklearn.metrics import roc_curve,auc

import os

import re

pd.set_option('precision',4) #设置小数点后面4位，默认是6位

matplotlib.rcParams['font.sans-serif'] = ['simHei'] #指定默认字体,防止乱码

matplotlib.rcParams['axes.unicode_minus'] = False #解决保存图像是负号'-'显示为方块的问题

matplotlib.rcParams['font.serif'] = ['SimHei']

font = {'family':'SimHei'}

matplotlib.rc(*('font',),**font)

#import warnings

#warnings.filterwarnings('ignore')

class DataProcess:

"""

About:数据预处理相关函数功能集

"""

def __init__(self,dep='y'): #默认是y

"""

about:初始参数

:param dep: str,the label of y;

"""

self.dep = dep

def str_var_check(self,df):

"""

about:移除df中的字符型变量

:param df: DataFrame

:return: DataFrame and 字符型变量名列表

"""

col_type = pd.DataFrame(df.dtypes) #获取每个变量的类型,返回Series

col_type.columns = ['type'] #增加column名称

save_col = list(col_type.loc[(col_type['type']== 'float64')|(col_type['type'] == 'int64')].index)

#得到是浮点型、数值型的列名

rm_col = [x for x in df.columns if x not in save_col] #得到df不含浮点型和数值的变量名

save_df = df[save_col] #得到浮点型和数值的变量数据

return (save_df,rm_col)

def df_dummy(self,df,col_list):

"""

about:变量离散化

:param df:DataFrame

:param col_list:list 需要dummy的变量列表

:return: DataFrame

"""

dummy_df = pd.get_dummies(df,prefix=None,prefix_sep='_',dummy_na=False,columns=col_list,sparse=False,drop_first=True) #转化哑变量,结果要转化的在后面，不需要转化的在前面

return dummy_df

def get_varname(self,df):

"""

about:获取特征变量名

:param df:DataFrame

:return DataFrame

"""

if self.dep in df.columns:

dfClearnVar = df.drop(self.dep,1)

else:

dfClearnVar = df

var_namelist = dfClearnVar.T

var_namelist['varname'] = var_namelist.index

var_namelist = var_namelist['varname'].reset_index()

var_namelist = var_namelist.drop('index',1)

return var_namelist

def calcConcentric(self,df):

"""

about:集中度计算

:calcuate the concentration rate

:mydat,pd.DataFrame

"""

sumCount = len(df.index)

colsDf = pd.DataFrame({'tmp':['tmp',np.nan,np.nan]})

for col in self.notempty_var_list: #此处的self.notempty_var_list在下面的var_status定义

print(col)

valueCountDict = {}

colDat = df.loc[:,col]

colValueCounts = pd.value_counts(colDat).sort_values(ascending=False) #按照colDat的值计数，再排序，按照大到小

concentElement = colValueCounts.index[0] #得到计算最大频率的值

valueCountDict[col] = [concentElement,colValueCounts.iloc[0],colValueCounts.iloc[0]* 1.0 / sumCount]# 得到值、出现频率、出现占总数据比

colDf = pd.DataFrame(valueCountDict)

colsDf = colsDf.join(colDf) #join 方法添加

#通过循环得到所有变量的取值的频率、占比

colsDf = (colsDf.rename(index={0:'concentricElement',1:'concentricCount',2:'concentricRate'})).drop('tmp',axis=1) #替换index的值，同时删除多余的tmp

return colsDf

def nmi(self,A,B):

"""

about:1、介绍NMI(Normalized Mutual Information)标准化互信息，常用在聚类中，度量两个聚类结果的相近程度。是社区发现(community detection)的重要衡量指标，

基本可以比较客观地评价出一个社区划分与标准划分之间相比的准确度。NMI的值域是0到1，越高代表划分得越准。

"""

B = B.fillna(-1)

total = len(A)

A_ids = set(A)

B_ids = set(B)

MI = 0

eps = 1.4e-45 #这个是1.4的负45次方

for idA in A_ids:

for idB in B_ids:

idAOccur = np.where(A == idA) #返回符合条件的坐标

idBOccur = np.where(B == idB)

idABOccur = np.intersect1d(idAOccur,idBOccur) #返回交集并排序

px = 1.0 * len(idAOccur[0]) / total

py = 1.0 * len(idBOccur[0]) / total

pxy = 1.0 * len(idABOccur) / total

MI = MI + pxy * math.log(pxy / (px * py) + eps,2)

Hx = 0

for idA in A_ids:

idAOccurCount = 1.0 * len(np.where(A == idA)[0])

Hx = Hx - idAOccurCount / total * math.log(idAOccurCount / total + eps,2)

Hy = 0

for idB in B_ids:

idBOccurCount = 1.0 * len(np.where(B == idB)[0])

Hy = Hy - idBOccurCount / total * math.log(idBOccurCount / total + eps,2)

MIhat = 2.0 * MI /(Hx + Hy)

return MIhat

def nmi_corr(self,df):

"""

about:返回所有预测数据和实际的相关系数和标准化互信息

"""

nmi_dict = {}

corr_dict = {}

for col in self.notempty_var_list:

nmi_value = self.nmi(df[self.dep],df[col]) #求标准化互信息

cor_value = np.corrcoef(df[self.dep],df[col])[(0,1)] #求相关系数，得到的结果是一个相关矩阵，取（0，1）和（1，0）是一样的

nmi_dict[col] = nmi_value

corr_dict[col] = cor_value

nmi_df = pd.DataFrame.from_dict(nmi_dict,orient='index') # orient='index'是以字典的键为INDEX，相当于行列置换

corr_df = pd.DataFrame.from_dict(corr_dict,orient='index')

nmi_corr_df = nmi_df.merge(corr_df,left_index=True,right_index=True) #以左右的index连接，默认是inner

nmi_corr_df.columns = ['nmi','corr']

return nmi_corr_df

def var_status(self,df):

"""

about:变量描述性统计

:param df:

:return: Dataframe

"""

print('正在进行变量描述统计...')

CleanVar = df.drop(self.dep,1)

describe = CleanVar.describe().T

self.notempty_var_list = list(describe.loc[describe['count'] > 0].index) #得到变量个数大于0的变量名

sample_num = int(describe['count'].max())

describe['varname'] = describe.index

describe.rename(columns={'count':'num','mean':'mean_v','std':'std_v','max':'max_v','min':'min_v'},inplace=True)

describe['saturation'] = describe['num'] / sample_num

describe = describe.drop(['25%','50%','75%'],1) #删除多余的列

describe = describe.reset_index(drop=True) #重设index，且放弃之前的index

describe['index'] = describe.index

print('正在计算变量集中度...')

Concent = self.calcConcentric(df)

concentricRate = pd.DataFrame(Concent.T['concentricRate'])

concentricRate['index'] = concentricRate.index

print('正在计算变量IV值...')

mywoeiv = Woe_iv(df,dep=self.dep,event=1,nodiscol=None,ivcol=None,disnums=20,X_woe_dict=None)

mywoeiv.woe_iv_vars()

iv = mywoeiv.iv_dict

iv = pd.DataFrame(pd.Series(iv,name='iv'))

iv['index'] = iv.index

var_des = pd.merge(describe,concentricRate,how='inner',left_on=['varname'],right_on=['index'])

var_des = pd.merge(var_des,iv,how='inner',left_on =['varname'],right_on=['index'])

var_des = var_des[['varname','num','mean_v','std_v','min_v','max_v','saturation','concentricRate','iv']]

print('描述统计完毕...')

return var_des

def var_filter(self,df,varstatus,min_saturation=0.01,max_concentricRate=0.98,min_iv=0.01):

"""

about:变量初筛

:param df:

:param varstatus: dataframe.var_status函数的输出结果

:param min_saturation:float 变量筛选值饱和度下限

:param max_concentricRate: folat 变量筛选值集中度上限

:param min_iv: float 变量筛选之IV值下限

:return: DataFrame 这里nan会变成false

"""

var_selected = list(varstatus['varname'].loc[(varstatus['saturation'] >= min_saturation) & (varstatus['concentricRate'] <= max_concentricRate)

& (varstatus['iv'] >= min_iv)]) #依据要求得到符合条件的变量名

var_selected.insert(0,self.dep) #插入y

df_selected = df[var_selected]

return df_selected #返回筛选后的数据

def var_corr_delete(self,df_WoeDone_select,var_desc_woe,corr_limit=0.95):

"""

about:剔除相关系数高的变量

:param df_WoeDone_select: 变量初删后返回的值

:param var_desc_woe: 变量描述统计后返回的值

:param corr_limit:

:return:

"""

deleted_vars = []

high_IV_var = list((df_WoeDone_select.drop(self.dep,axis=1)).columns)

for i in high_IV_var:

if i in deleted_vars:

continue

for j in high_IV_var:

if not i == j:

if not j in deleted_vars:

roh = np.corrcoef(df_WoeDone_select[i],df_WoeDone_select[j])[(0,1)] #比较每两个之间的相关系数，最后删除IV值较小的数

if abs(roh) > corr_limit:

x1_IV = var_desc_woe.iv.loc[var_desc_woe.varname == i].values[0] #这个就是IV值

y1_IV = var_desc_woe.iv.loc[var_desc_woe.varname == j].values[0]

if x1_IV > y1_IV:

deleted_vars.append(j)

print('变量' + i + '和' + '变量' + j + '高度相关,相关系数达到' + str(abs(roh)) + ',已删除' + j)

else:

deleted_vars.append(i)

print('变量' + i + '和' + '变量' + j + '高度相关,相关系数达到' + str(abs(roh)) + ',已删除' + i)

df_corr_select = df_WoeDone_select.drop(deleted_vars,axis=1) #删除选中的变量

print('已对相关系数达到' + str(corr_limit) + '以上的变量进行筛选，剔除的变量列表如下' + str(deleted_vars))

return df_corr_select

class Plot_vars:

"""

about:风险曲线图

cut_points_bring:单变量,针对连续变量离散化，目标是均分xx组，但当某一值重复过高时会适当调整，最后产出的是分割点，不包含首尾

dis_group:单变量，离散化连续变量，并生产一个groupby数据:

nodis_group:不需要离散化的变量，生产一个groupby数据:

plot_var:单变量绘图

plot_vars:多变量绘图

"""

def __init__(self,mydat,dep='y',nodiscol= None,plotcol=None,disnums=5,file_name=None):

"""

abount:变量风险图

:param model_name:

:param mydat:DataFrame,包含X,y的数据集

:param dep: str,the label of y

:param nodiscol: list,defalut None,当这个变量有数据时，会默认这里的变量不离散，且只画nodis_group,

其余的变量都需要离散化，且只画dis_group.当这个变量为空时，系统回去计算各变量的不同数值的数量，若小于15，则认为不需要离散，直接丢到nodiscol中

:param disnums: int,连续变量需要离散的组数

"""

self.mydat = mydat

self.dep = dep

self.plotcol = plotcol #这个是制定多个变量，批量跑多个变量

self.nodiscol = nodiscol

self.disnums = disnums

self.file_path = os.getcwd()

self.file_name = file_name

self.col_cut_points = {}

self.col_notnull_count = {}

for i in self.mydat.columns:

if i != self.dep:

self.col_cut_points[i] = []

for i in self.mydat.columns:

if i != self.dep:

col_notnull = len(self.mydat[i][pd.notnull(self.mydat[i])].index)

self.col_notnull_count[i] = col_notnull

if self.nodiscol is None:

nodiscol_tmp = []

for i in self.mydat.columns:

if i != self.dep:

col_cat_num = len(set(self.mydat[i][pd.notnull(self.mydat[i])]))

if col_cat_num < 5: #非空的数据分类小于5个

nodiscol_tmp.append(i)

if len(nodiscol_tmp) > 0:

self.nodiscol = nodiscol_tmp

if self.file_name is not None:

self.New_Path = self.file_path + '\\' + self.file_name + '\\'

if not os.path.exists(self.New_Path):

os.makedirs(self.New_Path) #增加新的文件夹

def cut_point_bring(self,col_order,col):

"""

about:分割函数

:param col_order:DataFrame,非null得数据集，包含y，按变量值顺序排列

:param col:str 变量名

:return:

"""

PCount = len(col_order.index)

min_group_num = self.col_notnull_count[col] / self.disnums #特定变量的非null数据量除以默认分组5组

disnums = int(PCount / min_group_num) #数据集的数量除以最小分组的数量

if PCount /self.col_notnull_count[col] >= 1 /self.disnums:

if disnums > 0 :

n_cut = int(PCount /disnums)

cut_point = col_order[col].iloc[n_cut - 1]

for i in col_order[col].iloc[n_cut:]:

if i == cut_point:

n_cut += 1

else:

self.col_cut_points[col].append(cut_point) #得到切点值

break #这里为了解决分割点的值多个相当，因此一直分到不相等，才退出

self.cut_point_bring(col_order[n_cut:],col) #这里是递归函数,用剩下的数据继续跑

def dis_group(self,col):

"""

abount:连续性变量分组

:param col:str,变量名称

:return:

"""

dis_col_data_notnull = self.mydat.loc[(pd.notnull(self.mydat[col]),[self.dep,col])]

Oreder_P = dis_col_data_notnull.sort_values(by=[col],ascending=True)

self.cut_point_bring(Oreder_P,col)

dis_col_cuts = []

dis_col_cuts.append(dis_col_data_notnull[col].min()) #得到变量的最小值

dis_col_cuts.extend(self.col_cut_points[col]) #加入变量的切点值

dis_col_data = self.mydat.loc[:,[self.dep,col]]

dis_col_data['group'] = np.nan

for i in range(len(dis_col_cuts) - 1): #这里开始依据分组的切点，改变group的值

if i == 0:

dis_col_data.loc[dis_col_data[col] <= dis_col_cuts[i+1],['group']] = i

elif i == len(dis_col_cuts) - 2:

dis_col_data.loc[dis_col_data[col] > dis_col_cuts[i],['group']] = i

else:

dis_col_data.loc[(dis_col_data[col] > dis_col_cuts[i]) & (dis_col_data[col] <= dis_col_cuts[i+1]),['group']] = i

dis_col_data[col] = dis_col_data['group']

dis_col_bins = []

dis_col_bins.append('nan')

dis_col_bins.extend(['(%s,%s]' % (dis_col_cuts[i],dis_col_cuts[i+1]) for i in range(len(dis_col_cuts) - 1)])

dis_col = dis_col_data.fillna(-1)

col_group = (dis_col.groupby([col],as_index=False))[self.dep].agg({'count':'count','bad_num':'sum'}) #按照切点分组，按dep计数

avg_risk = self.mydat[self.dep].sum() /self.mydat[self.dep].count() #平均风险度

col_group['per'] = col_group['count'] / col_group['count'].sum() #占比

col_group['bad_per'] = col_group['bad_num'] / col_group['count'] #坏客户占比

col_group['risk_times'] = col_group['bad_per'] / avg_risk #风险倍数

if -1 in list(col_group[col]):

col_group['bins'] = dis_col_bins

else:

col_group['bins'] = dis_col_bins[1:]

col_group[col] = col_group[col].astype(np.float) #转换数据类型

col_group = col_group.sort_values([col],ascending=True)

col_group = pd.DataFrame(col_group,columns=[col,'bins','per','bad_per','risk_times'])

return col_group

def nodis_group(self,col):

"""

aount:离散型变量分组，主要处理null值

:param col:str 变量名称

:return:

"""

nodis_col_data = self.mydat.loc[:,[self.dep,col]]

is_na = pd.isnull(nodis_col_data[col]).sum() > 0 #判断是否有null

col_group = (nodis_col_data.groupby([col],as_index=False))[self.dep].agg({'count':'count','bad_num':'sum'})

col_group = pd.DataFrame(col_group,columns=[col,'bad_num','count'])

if is_na:

y_na_count = nodis_col_data[pd.isnull(nodis_col_data[col])][self.dep].count()

y_na_sum = nodis_col_data[pd.isnull(nodis_col_data[col])][self.dep].sum()

col_group.loc[len(col_group.index),:] = [-1,y_na_sum,y_na_count] #添加到最后一行

avg_risk = self.mydat[self.dep].sum() /self.mydat[self.dep].count()

col_group['per'] = col_group['count'] / col_group['count'].sum()

col_group['bad_per'] = col_group['bad_num'] /col_group['count']

col_group['risk_times'] = col_group['bad_per'] / avg_risk #风险倍数

if is_na:

bins = col_group[col][:len(col_group.index) - 1]

bins[len(col_group.index)] = 'nan'

col_group['bins'] = bins

col_labels = list(range(len(col_group.index) - 1))

col_labels.append(-1)

col_group[col] = col_labels

else:

bins = col_group[col]

col_group['bins'] = bins

col_labels = list(range(len(col_group.index)))

col_group[col] = col_labels

col_group[col] = col_group[col].astype(np.float)

col_group = col_group.sort_values([col],ascending=True)

col_group = pd.DataFrame(col_group,columns=[col,'bins','per','bad_per','risk_times'])

return col_group

def plot_var(self,col):

"""

about:单变量泡泡图

:param col: str 变量名称

:return:

"""

if self.nodiscol is not None:

if col in self.nodiscol:

col_group = self.nodis_group(col)

else:

col_group = self.dis_group(col)

fix,ax1 = plt.subplots()

ax1.bar(x=list(col_group[col]),height=col_group['per'],width=0.6,align='center') #bar是柱形图

for a,b in zip(col_group[col],col_group['per']):

ax1.text(a,b+0.005,'%.4f' %b,ha='center',va='bottom',fontsize=8) #给柱形图加上文字标示

ax1.set_xlabel(col)

ax1.set_ylabel('percent',fontsize=12)

ax1.set_xlim([-2,max(col_group[col])+2]) #设置刻度长度

ax1.set_ylim([0,max(col_group['per'])+0.3])

ax1.grid(False) #不显示网格

ax2 = ax1.twinx() #在原有的图像上添加坐标轴，类似双轴

ax2.plot(list(col_group[col]),list(col_group['risk_times']),'-ro',color='red')

for a,b in zip(col_group[col],col_group['risk_times']):

ax2.text(a,b+0.05,'%.4f' % b,ha='center',va='bottom',fontsize=7)

ax2.set_ylabel('risktimes',fontsize=12)

ax2.set_ylim([0,max(col_group['risk_times']) + 0.5])

ax2.grid(False)

plt.title(col)

the_table = plt.table(cellText=col_group.round(4).values,colWidths=[0.12]*len(col_group.columns), #round(4)保留4位小数

rowLabels=col_group.index,colLabels=col_group.columns,loc=1) #loc=1显示在右对齐.loc=2显示在左对齐.colWidths是每列表格的长度占比，5列，每列0.2，就占满

the_table.auto_set_font_size(False) #自动设置字体大小

the_table.set_fontsize(10) #设置表格的大小

if self.file_path is not None:

plt.savefig(self.New_Path+col+'.pdf',dpi=600)

# plt.show()

def plot_vars(self):

"""

about:批量跑多个变量，如果未制定多列或者单个变量，将会跑所有的变量

"""

if self.plotcol is not None: #这个是制定多个变量，批量跑多个变量

for col in self.plotcol:

self.plot_var(col)

else:

cols = self.mydat.columns

for col in cols:

print(col)

if col != self.dep:

self.plot_var(col)

class Woe_iv:

"""

about: woe iv 类计算函数

check_target_binary:检查是否是二分类问题

target_count:计算好坏样本数目

cut points_bring:单变量，针对连续变量离散化，目标是均分xx组，但当某一数值重复过高时会适当调整，最后产出的是分割点，不包含首尾

dis_group: 单变量，离散化连续变量，并生产一个groupby数据

nodis_group:不需要离散化的变量，生产一个groupby数据

woe_iv_var:单变量，计算各段的woe值和iv

woe_iv_vars:多变量，计算多变量的woe和iv

apply_woe_replace:将数据集中的分段替换成对应的woe值

一般应大于0.02，默认选IV大于0.1的变量进模型，但具体要结合实际。如果IV大于0.5，就是过预测（over-predicting）变量

AUC在 0.5～0.7时有较低准确性， 0.7～0.8时有一定准确性, 0.8~0.9则高，AUC在0.9以上时有非常高准确性。AUC=0.5时，说明诊断方法完全不起作用，无诊断价值

psi 判断：index <= 0.1，无差异；0.1< index <= 0.25，需进一步判断；0.25 <= index，有显著位移，模型需调整。

"""

def __init__(self,mydat,dep='y',event=1,nodiscol=None,ivcol=None,disnums=20,X_woe_dict=None):

"""

about:初始化参数设置

:param mydat:DataFrame 输入的数据集，包含y

:param dep: str,the label of y

:param event: int y中bad的标签

:param nodiscol: list,defalut

None,不需要离散的变量名，当这个变量有数据时，会默认这里的变量不离散，且只跑nodis_group,其余的变量都需要离散化，且只跑

dis_group。当这个变量为空时，系统会去计算各变量的不同数值的数量，若小于15，则认为不需要离散，直接丢到nodiscol中

:param ivcol: list 需要计算woe,iv的变量名，该变量不为None时，只跑这些变量，否则跑全体变量

:param disnums: int 连续变量离散化的组数

:param X_woe_dict: dict 每个变量每段的woe值，这个变量主要是为了将来数据集中的分段替换成对应的woe值

即输入的数据集已经超过离散化分段处理，只需要woe化而已

"""

self.mydat = mydat

self.event = event

self.nodiscol = nodiscol

self.ivcol = ivcol

self.disnums = disnums

self._WOE_MIN = -20

self._WOE_MAX = 20

self.dep = dep

self.col_cut_points = {}

self.col_notnull_count = {}

self._data_new = self.mydat.copy(deep=True)

self.X_woe_dict = X_woe_dict

self.iv_dict = None

for i in self.mydat.columns:

if i != self.dep:

self.col_cut_points[i] = []

for i in self.mydat.columns:

if i != self.dep:

col_notnull = len(self.mydat[i][pd.notnull(self.mydat[i])].index)

self.col_notnull_count[i] = col_notnull

if self.nodiscol is None:

nodiscol_tmp = []

for i in self.mydat.columns:

if i != self.dep:

col_cat_num = len(set(self.mydat[i][pd.notnull(self.mydat[i])]))

if col_cat_num < 20:

nodiscol_tmp.append(i)

if len(nodiscol_tmp) > 0:

self.nodiscol = nodiscol_tmp

def check_target_binary(self,y):

"""

about: 检测因变量是否为二元变量

:param y: the target variable,series type

:return:

"""

y_type = type_of_target(y) #检测数据是不是二分类数据

if y_type not in ('binary',):

raise ValueError('Label tyoe must be binary!!!')

def target_count(self,y):

"""

about：计算Y值得数量

:param y: the target variable,series type

:return: 0,1的数量

"""

y_count = y.value_counts()

if self.event not in y_count.index:

event_count = 0

else:

event_count = y_count[self.event]

non_event_count = len(y) - event_count

return (event_count,non_event_count) #返回好坏客户的数量

def cut_points_bring(self,col_order,col):

"""

about:分割函数

:param col_order: DataFrame 非null的数据集，包含y，按变量值顺序排列

:param col: str 变量名

:return

"""

PCount = len(col_order.index)

min_group_num = self.col_notnull_count[col] /self.disnums

disnums = int(PCount/min_group_num)

if PCount / self.col_notnull_count[col] >= 1 / self.disnums:

if disnums >0 :

n_cut = int(PCount / disnums)

cut_point = col_order[col].iloc[n_cut-1]

for i in col_order[col].iloc[n_cut:]:

if i == cut_point:

n_cut += 1

else:

self.col_cut_points[col].append(cut_point)

break

self.cut_points_bring(col_order[n_cut:],col)

def dis_group(self,col):

"""

abount:连续型变量分组

:param col:str 变量名称

:return:

"""

dis_col_data_notnull = self.mydat.loc[(pd.notnull(self.mydat[col]),[self.dep,col])]

Oreder_P = dis_col_data_notnull.sort_values(by=[col],ascending=True)

self.cut_points_bring(Oreder_P,col)

dis_col_cuts = []

dis_col_cuts.append(dis_col_data_notnull[col].min()) #得到变量的最小值

dis_col_cuts.extend(self.col_cut_points[col]) #加入变量的切点值

dis_col_cuts.append(dis_col_data_notnull[col].max()) #得到变量的最大值

dis_col_data = self.mydat.loc[:,[self.dep,col]]

dis_col_data['group'] = np.nan

for i in range(len(dis_col_cuts) - 1): #这里开始依据分组的切点，改变group的值

if i == 0:

dis_col_data.loc[dis_col_data[col] <= dis_col_cuts[i+1],['group']] = i

elif i == len(dis_col_cuts) - 2:

dis_col_data.loc[dis_col_data[col] > dis_col_cuts[i],['group']] = i

else:

dis_col_data.loc[(dis_col_data[col] > dis_col_cuts[i]) & (dis_col_data[col] <= dis_col_cuts[i+1]),['group']] = i

dis_col_data[col] = dis_col_data['group']

dis_col_bins = []

dis_col_bins.append('nan')

dis_col_bins.extend(['(%s,%s]' % (dis_col_cuts[i],dis_col_cuts[i+1]) for i in range(len(dis_col_cuts) - 1)])

dis_col = dis_col_data.fillna(-1)

col_group = (dis_col.groupby([col],as_index=False))[self.dep].agg({'count':'count','bad_num':'sum'}) #按照切点分组，按dep计数

col_group['per'] = col_group['count'] / col_group['count'].sum() #占比

col_group['good_num'] = col_group['count'] - col_group['bad_num'] #好客户数量

if -1 in list(col_group[col]):

col_group['bins'] = dis_col_bins

else:

col_group['bins'] = dis_col_bins[1:]

for i in range(len(dis_col_cuts) - 1):

if i == 0:

self._data_new.loc[self.mydat[col] <= dis_col_cuts[i + 1],[col]] = dis_col_bins[i + 1]

else:

if i == len(dis_col_cuts) - 2:

self._data_new.loc[self.mydat[col] > dis_col_cuts[i],[col]] = dis_col_bins[i + 1]

else:

self._data_new.loc[(self.mydat[col] > dis_col_cuts[i]) & (self.mydat[col] <= dis_col_cuts[i + 1]),[col]] = dis_col_bins[i + 1]

self._data_new[col].fillna(value='nan',inplace=True)

return col_group

def nodis_group(self,col):

"""

about:离散型变量分组

:param col: str 变量名称

:return:

"""

nodis_col_data = self.mydat.loc[:,[self.dep,col]]

is_na = pd.isnull(nodis_col_data[col]).sum() > 0 #判断是否有null

col_group = (nodis_col_data.groupby([col],as_index=False))[self.dep].agg({'count':'count','bad_num':'sum'})

col_group = pd.DataFrame(col_group,columns=[col,'bad_num','count'])

if is_na:

y_na_count = nodis_col_data[pd.isnull(nodis_col_data[col])][self.dep].count()

y_na_sum = nodis_col_data[pd.isnull(nodis_col_data[col])][self.dep].sum()

col_group.loc[len(col_group.index),:] = [-1,y_na_sum,y_na_count] #添加到最后一行

col_group['per'] = col_group['count'] / col_group['count'].sum()

col_group['good_num'] = col_group['count'] - col_group['bad_num']

if is_na:

bins = col_group[col][:len(col_group.index) - 1]

bins.loc[len(bins.index)] = 'nan'

col_group['bins'] = bins

col_labels = list(range(len(col_group.index) - 1))

col_labels.append(-1)

col_group[col] = col_labels

else:

bins = col_group[col]

col_group['bins'] = bins

col_labels = list(range(len(col_group.index)))

col_group[col] = col_labels

return col_group

def woe_iv_var(self,col,adj=1):

"""

about:单变量woe分布及iv值计算

:param col: str 变量名称

:param adj: float 分母为0时的异常调整

:return:

"""

self.check_target_binary(self.mydat[self.dep]) #检查目标值是否时二分类

event_count,non_event_count = self.target_count(self.mydat[self.dep]) #计算坏、好客户数量

if self.nodiscol is not None:

if col in self.nodiscol:

col_group = self.nodis_group(col)

else:

col_group = self.dis_group(col)

x_woe_dict = {}

iv = 0

for cat in col_group['bins']:

cat_event_count = col_group.loc[(col_group.loc[:,'bins'] == cat,'bad_num')].iloc[0] #分组中坏客户的数量

cat_non_event_count = col_group.loc[(col_group.loc[:,'bins'] == cat,'good_num')].iloc[0] #分组中好客户的数量

rate_event = cat_event_count * 1.0 / event_count #本组的坏客户/总的坏客户

rate_non_event = cat_non_event_count * 1.0 / non_event_count #本周的好客户/总的好客户

if rate_non_event == 0:#这个是让分子、分母都不为0

woe1 = math.log((cat_event_count * 1.0 + adj) / event_count / ((cat_non_event_count * 1.0 + adj) / non_event_count)) #本组坏客户占比除以好客户占比

else:

if rate_event == 0:

woe1 = math.log((cat_event_count * 1.0 +adj) / event_count / ((cat_non_event_count * 1.0 + adj) / non_event_count))

else:

woe1 = math.log(rate_event / rate_non_event)

x_woe_dict[cat] = woe1

iv += abs((rate_event - rate_non_event) * woe1)

return (x_woe_dict,iv)

def woe_iv_vars(self,adj=1):

"""

about:多变量woe分布及IV值计算

:param adj:folat,分母为0时的异常调整

:return:

"""

X_woe_dict = {}

iv_dict = {}

if self.ivcol is not None: #ivcol这里也是制定批量处理的

for col in self.ivcol:

print(col)

x_woe_dict,iv = self.woe_iv_var(col,adj)

X_woe_dict[col] = x_woe_dict

iv_dict[col] = iv

else:

for col in self.mydat.columns: #这里时全部处理

print(col)

if col != self.dep:

x_woe_dict,iv = self.woe_iv_var(col,adj)

X_woe_dict[col] = x_woe_dict

iv_dict[col] = iv

self.X_woe_dict = X_woe_dict #储存IV和WOE值

self.iv_dict = iv_dict

def apply_woe_replace(self):

"""

about:变量woe值替换

:return：

"""

for col in self.X_woe_dict.keys():#这个格式是字典中带字典

for binn in self.X_woe_dict[col].keys():

self._data_new.loc[(self._data_new.loc[:,col] == binn,col)] = self.X_woe_dict[col][binn]

self._data_new = self._data_new.astype(np.float64) #得到_data_new是所有分箱后的woe值

def woe_dict_save(self,obj,woe_loc):

"""

about:woe 字典保存

:param obj:

:param woe_loc:保存路径及文件名

:return:

"""

with open(woe_loc,'wb') as f:

pickle.dump(obj,f,pickle.HIGHEST_PROTOCOL) #将obj对象保存在f中，HIGHEST_PROTOCOL是整型，最高协议版本

def woe_dict_load(self,woe_loc):

"""

about: woe 字典读取

:param woe_loc:读取路径及文件名

:return:

"""

with open(woe_loc,'rb') as f:

return pickle.load(f)

class BestBin:

"""

about:分箱类函数

"""

def __init__(self,data_in,dep='y',method=4,group_max=3,per_limit=0.05,min_count=0.05):

"""

about:参数设置

:param data_in:DataFrame 包含X,y的数据集

:param dep:str,the label of y;

:param method:int,defaut 4,分箱指标选取：1 基尼系数，2 信息熵，3 皮尔逊卡方统计量、4IV值

:param group_max: int,defalut 3, 最大分箱组数

:param per_limit: float,小切割范围占比

"""

self.data_in = data_in

self.dep = dep

if self.dep in self.data_in.columns:

self.CleanVar = self.data_in.drop(self.dep,1) #删除Y

else:

self.CleanVar = self.data_in

self.method = method

self.group_max = group_max

self.per_limit = per_limit

self.min_count = min_count

self.file_path = os.getcwd()

self.bin_num = int(1 / per_limit)

self.nrows = data_in.shape[0]

self.col_cut_points = {}

self.col_notnull_count = {}

for i in self.CleanVar.columns:

self.col_cut_points[i] = []

col_notnull = len(self.data_in[i][pd.notnull(self.data_in[i])].index)

self.col_notnull_count[i] = col_notnull

def cut_points_bring(self,col_order,col):

"""

about:分割函数

:param col_order:DataFrame,非null的数据集，包含y，按变量值顺序排列

:param col: str 变量名

:return:

"""

PCount = len(col_order.index) #得到数据长度

min_group_num = self.col_notnull_count[col] * self.per_limit #得到最小分组数量，这个是不变的

disnums = int(PCount/min_group_num) #得到分组数，每次都会变化

if PCount / self.col_notnull_count[col] >= self.per_limit * 2: #剩余的数量/总的数量 大于等于最小分割的2呗，留下最后一份

if disnums > 0 : #最后分组要大于0

n_cut = int(PCount / disnums)

cut_point = col_order[col].iloc[n_cut-1]

for i in col_order[col].iloc[n_cut:]:

if i == cut_point:

n_cut += 1

else:

self.col_cut_points[col].append(cut_point)

break

self.cut_points_bring(col_order[n_cut:],col) #递归调用自己

def groupbycount(self,input_data,col):

"""

abount:分组

:param col:str 变量名称

:param input_data:DataFrame 输入数据

:return:

"""

dis_col_data_notnull = self.data_in.loc[(pd.notnull(self.data_in[col]),[self.dep,col])] #得到变量函和Y的数据

Oreder_P = dis_col_data_notnull.sort_values(by=[col],ascending=True) #将所有的数据排序

self.cut_points_bring(Oreder_P,col) #调用分割函数,按照

dis_col_cuts = []

dis_col_cuts.append(dis_col_data_notnull[col].min()) #得到变量的最小值

dis_col_cuts.extend(self.col_cut_points[col]) #加入变量的切点值

dis_col_cuts.append(dis_col_data_notnull[col].max()) #加入变量最大值

input_data['group'] = input_data[col]

for i in range(len(dis_col_cuts) - 1): #这里开始依据分组的切点，改变group的值

if i == 0: # 第一组

input_data.loc[input_data[col] <= dis_col_cuts[i+1],['group']] = i + 1

elif i == len(dis_col_cuts) - 2: # 最后一组

input_data.loc[input_data[col] > dis_col_cuts[i],['group']] = i + 1

else:

input_data.loc[(input_data[col] > dis_col_cuts[i]) & (input_data[col] <= dis_col_cuts[i+1]),['group']] = i + 1

return (input_data,dis_col_cuts) #返回分组值，分组切点

def bin_con_var(self,data_in_stable,indep_var,group_now):

"""

about:分箱

:param data_in_stable:输入数据集

:param indep_var:自变量

:param group_now:分箱切点映射表,也是那个最大分组数

:return:

"""

data_in = data_in_stable.copy() #复制

data_in,bin_cut = self.groupbycount(data_in,indep_var) #调用分组函数

Mbins = len(bin_cut) - 1

Temp_DS = data_in.loc[:,[self.dep,indep_var,'group']]

Temp_DS['group'].fillna(0, inplace=True) #将null值用0来填充-未实施此步骤

temp_blimits = pd.DataFrame({'Bin_LowerLimit':[],'Bin_UpperLimit':[],'group':[]})

for i in range(Mbins): #得到变量分组后的上下限值

temp_blimits.loc[(i,['Bin_LowerLimit'])] = bin_cut[i]

temp_blimits.loc[(i,['Bin_UpperLimit'])] = bin_cut[i+1]

temp_blimits.loc[(i,['group'])] = i + 1

grouped = Temp_DS.loc[:,[self.dep,'group']].groupby(['group']) #分组

g1 = grouped.sum().rename(columns={self.dep:'y_sum'})

g2 = grouped.count().rename(columns={self.dep:'count'})

g3 = pd.merge(g1,g2,left_index=True,right_index=True)

g3['good_sum'] = g3['count'] - g3['y_sum']

g3['group'] = g3.index

g3['PDVI'] = g3.index

g3.rename(columns={'group': 'id'},inplace=True) # 此处是后来有问题增加的

temp_cont = pd.merge(g3,temp_blimits,how='left',on='group') #得到分组统计的数据

for i in range(Mbins):

mx = temp_cont.loc[(temp_cont['group'] == i+1,'group')].values

if mx:

Ni1 = temp_cont['good_sum'].loc[temp_cont['group'] == i+1].values[0]

Ni2 = temp_cont['y_sum'].loc[temp_cont['group'] == i+1].values[0]

count = temp_cont['count'].loc[temp_cont['group'] == i+1].values[0]

bin_lower = temp_cont['Bin_LowerLimit'].loc[temp_cont['group'] == i+1].values[0]

bin_upper = temp_cont['Bin_UpperLimit'].loc[temp_cont['group'] == i+1].values[0]

if i == Mbins - 1: #如果i是最后一位

i1 = temp_cont['group'].loc[temp_cont['group'] < Mbins].values.max() #最后一位就取前面中最大的

else:

i1 = temp_cont['group'].loc[temp_cont['group'] > i+1].values.min() #其他的就取后面中最小的

if Ni1 == 0 or Ni2 == 0 or count == 0:

#如果有一组好客户、或者坏客户、或者总数为0，就把本组的所有人数加到后一组中，如果是最后一组，就加到上一组中，同时改正分组的阈值，再删除当组的数据

temp_cont['good_sum'].loc[temp_cont['group'] == i1] = temp_cont['good_sum'].loc[temp_cont['group'] == i1].values[0] + Ni1

temp_cont['y_sum'].loc[temp_cont['group'] == i1] = temp_cont['y_sum'].loc[temp_cont['group'] == i1].values[0] + Ni2

temp_cont['count'].loc[temp_cont['group'] == i1] = temp_cont['count'].loc[temp_cont['group'] == i1].values[0] + count

if i < Mbins -1:

temp_cont['Bin_LowerLimit'].loc[temp_cont['group'] == i1] = bin_lower

else:

temp_cont['Bin_UpperLimit'].loc[temp_cont['group'] == i1] = bin_upper

delete_indexs = list(temp_cont.loc[temp_cont['group'] == i+1].index)

temp_cont = temp_cont.drop(delete_indexs)

temp_cont['new_index'] = (temp_cont.reset_index(drop=True)).index + 1

temp_cont['var'] = temp_cont['group']

temp_cont['group'] = 1

Nbins = 1

while Nbins < group_now: #这会一直分组

Temp_Splits = self.CandSplits(temp_cont)

temp_cont = Temp_Splits

Nbins = Nbins + 1

temp_cont.rename(columns={'var':'OldBin'},inplace=True)

temp_Map1 = temp_cont.drop(['good_sum','PDVI','new_index'],axis=1)

temp_Map1 = temp_Map1.sort_values(by=['group'])

min_group = temp_Map1['group'].min()

max_group = temp_Map1['group'].max()

lmin = temp_Map1['Bin_LowerLimit'].min()

notnull = temp_Map1.loc[temp_Map1['Bin_LowerLimit'] > lmin - 10]

var_map = pd.DataFrame({'group':[],'LowerLimit':[],'UpperLimit':[],'count':[],'risk':[]})

for i in range(min_group,max_group+1):

ll = notnull['Bin_LowerLimit'].loc[notnull['group'] == i].min()

uu = notnull['Bin_UpperLimit'].loc[notnull['group'] == i].max()

con = notnull['count'].loc[notnull['group'] == i].sum()

yy = notnull['y_sum'].loc[notnull['group'] == i].sum()

if con > 0:

risk = yy * 1.0 / (con + 0.0001)

var_map = var_map.append({'group':i,'LowerLimit':ll,'UpperLimit':uu,'count':con,'risk':risk},ignore_index=True)

null_group = temp_Map1['group'].loc[temp_Map1['Bin_LowerLimit'].isnull()]

if null_group.any():

temp_Map_null = temp_Map1.loc[temp_Map1['Bin_LowerLimit'].isnull()]

ll = temp_Map_null['Bin_LowerLimit'].min()

uu = temp_Map_null['Bin_UpperLimit'].max()

con = temp_Map_null['count'].sum()

yy = temp_Map_null['y_sum'].sum()

i = temp_Map_null['group'].max()

risk = yy * 1.0 / con

var_map = var_map.append({'group':i,'LowerLimit':ll,'UpperLimit':uu,'count':con,'risk':risk},ignore_index=True)

var_map = var_map.sort_values(by=['LowerLimit','UpperLimit'])

var_map['newgroup'] = var_map.reset_index().index + 1

var_map = var_map.reset_index()

var_map = var_map.drop('index',1)

if null_group.any():

ng = var_map['group'].loc[var_map['LowerLimit'].isnull()].max()

notnull = var_map.loc[var_map['LowerLimit'].notnull()]

cng = var_map['group'].loc[var_map['group'] == ng].count()

if cng > 1:

var_map['newgroup'].loc[var_map['LowerLimit'].isnull()] = notnull['newgroup'].loc[notnull['group'] == ng].max()

var_map['group'] = var_map['newgroup']

else:

var_map['group'] = var_map['newgroup']

var_map['group'].loc[var_map['LowerLimit'].isnull()] = 0

else:

var_map['group'] = var_map['newgroup']

var_map = var_map.drop('newgroup',1)

var_map['per'] = var_map['count'] * 1.0 / var_map['count'].sum()

return var_map

def df_bin_con_var(self):

"""

about:分箱

:return

"""

ToBin = self.CleanVar.T #删除后的Y转置

ToBin['varname'] = ToBin.index #得到变量名

for i in ToBin['varname']:

print(i)

varnum = self.data_in.loc[:,[self.dep,i]].groupby([i]) #以变量分组,groupby 自动删除null计数

vcount = len(varnum.count().index) #得到分组的个数

sum_vcount = sum(varnum.count().index.tolist()) #这里判断是否是2分类，且只有0和1

if vcount <= 2:

self.data_in.loc[:,i + '_g'] = self.data_in[i]

self.data_in.loc[self.data_in[i + '_g'].isnull(),[i + '_g']] = -1 #将_g变量中所有的null变为-1

else:

var_group_map = self.bin_con_var(self.data_in,i,self.group_max) #开始分组,得到分组值

mincount = var_group_map['per'].min() #得到占比最小

group_now = len(var_group_map.group.unique()) #得到分组数

while mincount < self.min_count and group_now > 3:#判断最小分组占比小于设定的比率，且现在分组大于2组，则一直分下去

group_now = group_now - 1

var_group_map = self.bin_con_var(self.data_in,i,group_now)

mincount = var_group_map['count'].min() / var_group_map['count'].sum()

self.ApplyMap(self.data_in,i,var_group_map) #

print(var_group_map)

if self.file_path is not None:

self.New_Path = self.file_path + '\\bestbin\\'

if not os.path.exists(self.New_Path):

os.makedirs(self.New_Path)

var_group_map.to_csv(self.New_Path + i + '.csv')

def CandSplits(self,BinDS):

"""

about:Generate all candidate splits from currentBins and select the best new bins,first we sort the dataset OldBins by PDVI and Bin

:param BinDS

:return

"""

BinDS.sort_values(by=['group','PDVI'],inplace=True) #先排序

Bmax = BinDS['group'].values.max() #取分组最大

m = []

names = locals() #返回当前的所有局部变量

for i in range(Bmax): #当两组之后的，就把数据集合给分割成不同的部分

names['x%s' % i] = BinDS.loc[BinDS['group'] == i + 1]

m.append(BinDS['group'].loc[BinDS['group'] == i + 1].count()) #不同部分的组别个数，大于1下面就分割，否则不分割

temp_allVals = pd.DataFrame({'BinToSplit':[],'DatasetName':[],'Value':[]})

for i in range(Bmax): # 对每个组别

if m[i] > 1: # 如果组别的个数大于1

testx = self.BestSplit(names['x%s' % i],i) #调用函数

names['temp_trysplit%s' % i] = testx

names['temp_trysplit%s' % i].loc[names['temp_trysplit%s' % i]['Split'] == 1,['group']] = Bmax +1

d_indexs = list(BinDS.loc[BinDS['group'] == i + 1].index)

names['temp_main%s' % i] = BinDS.drop(d_indexs)

names['temp_main%s' % i] = pd.concat([names['temp_main%s' % i], names['temp_trysplit%s' % i]])

Value = self.GValue(names['temp_main%s' % i]) #这里是求按照上述分组后的值

temp_allVals = temp_allVals.append({'BinToSplit':i,'DatasetName':'temp_main%s' % i,'Value':Value},ignore_index=True)

#这里是对分了两组后的数据，再此分组，加入其的Value值，下面来判断哪个是最大的，那下次返回的数据就是最大的一个的分组，这样1分为2，2分为3，不会到4

ifsplit = temp_allVals['BinToSplit'].max()

if ifsplit >= 0:

temp_allVals = temp_allVals.sort_values(by=['Value'],ascending=False)

bin_i = int(temp_allVals['BinToSplit'][0])

NewBins = names['temp_main%s' % bin_i].drop('Split',1)

else:

NewBins = BinDS

return NewBins

def BestSplit(self,BinDs,BinNo):

"""

about:

:param BinDs

:param BinNo

:return:

"""

mb = BinDs['group'].loc[BinDs['group'] == BinNo +1].count()

BestValue = 0

BestI = 1

for i in range(mb - 1):#重复循环，计算出最大的Value值

Value = self.CalcMerit(BinDs,i+1) #再次调用函数,以i+1为分割点，计算相应两部分的值

if BestValue < Value:

BestValue = Value

BestI = i + 1

BinDs.loc[:,'Split'] = 0

BinDs.loc[BinDs['new_index'] <= BestI,['Split']] = 1 #可以找到最佳的二分点

BinDs = BinDs.drop('new_index',1)

BinDs = BinDs.sort_values(by=['Split','PDVI'],ascending=True)

BinDs['testindex'] = (BinDs.reset_index(drop=True)).index

BinDs['new_index'] = BinDs['testindex'] + 1

BinDs.loc[BinDs['Split'] == 1,['new_index']] = BinDs['new_index'].loc[BinDs['Split'] == 1] - BinDs['Split'].loc[BinDs['Split'] == 0].count()

NewBinDs = BinDs.drop('testindex',1)

return NewBinDs

def CalcMerit(self,BinDs,ix):

"""

about:

:param BinDs

:param ix

:return

"""

n_11 = BinDs['good_sum'].loc[BinDs['new_index'] <= ix].sum()

n_21 = BinDs['good_sum'].loc[BinDs['new_index'] > ix].sum()

n_12 = BinDs['y_sum'].loc[BinDs['new_index'] <= ix].sum()

n_22 = BinDs['y_sum'].loc[BinDs['new_index'] > ix].sum()

n_1s = BinDs['count'].loc[BinDs['new_index'] <= ix].sum()