[Py3.5] 데이터 전처리 및 기술 통계 분석

/*******************************************************************************************************************
-- Title : [Py3.5] 데이터 전처리 및 기술 통계 분석
-- Reference : acorn, googling
-- Key word : data pre-processing .csv csv api urllib api read_csv hist histogram boxplot cdf probplot
                  확률 도표 히스토그램 박스플롯 박스 플롯 누적 분포 함수 누적분포함수 scipy 와이블 분포
                  cumulative distributioin function weibull distribution weibull_min rectangle errorbar 
                  matplotlib pyplot numpy pandas scipy patches transforms provability plot
*******************************************************************************************************************/

■ Figure

■ Scripts

# -*- coding: utf-8 -*-
 
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats as st
import urllib.request
import os
import matplotlib.patches as patches
import matplotlib.transforms as transforms
 
# ------------------------------
# -- Set Dataframe Option
# ------------------------------
pd.set_option('display.height', 1000)
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
 
 
# ------------------------------
# -- Get the data & Save file
# ------------------------------
 
# -- Select current path
print (os.getcwd())
print ("... current_path", "." * 100, "\n")
 
# --
# -- Get the .CSV data via API
# --
# API 호출 후 바로 저장 
suicide_rate_url = 'http://apps.who.int/gho/athena/data/xmart.csv?target=GHO/MH_12&profile=crosstable&filter=COUNTRY:*;REGION:*&x-sideaxis=COUNTRY&x-topaxis=GHO;YEAR;SEX'
local_filename, headers = urllib.request.urlretrieve(suicide_rate_url, filename='who_suicide_rates.csv')
 
print ("[local_filename]:", local_filename)
print ("[headers]:", headers)
print ("... get_api_csv", "." * 100, "\n")
 
 
# --
# -- Read the .CSV data
# --
path_csv = "D:\\PyProject\\20170524_acorn1\\who_suicide_rates.csv"
df_rates = pd.read_csv(path_csv, skiprows=2)
 
print (df_rates.head())
print ("... read_csv", "." * 100, "\n")
 
# --
# -- Alter column
# --
df_rates = df_rates[["Country", " Both sexes", " Male", " Female"]]
df_rates.columns = ["Country", "Both", "Male", "Female"]
 
print (df_rates.head())
print ("... alter_column", "." * 100, "\n")
 
 
# ------------------------------
# -- Basic Stats. Analysis
# ------------------------------
 
# --
# -- Draw histogram
# --
df_rates.plot.hist(stacked=True, y=['Male', 'Female'], bins=30, color=['Coral', 'Green'])
plt.xlabel("xlabel")
plt.ylabel("ylabel")
plt.title("Histogram Graph Title")
plt.legend()
 
plt.show()
 
print (",,, draw_histogram", "," * 100, "\n")
 
# --
# -- Draw boxplot
# --
print (df_rates['Male'].mean(), df_rates['Female'].mean())
 
df_rates.boxplot(['Both', 'Male','Female'])
plt.xlabel("xlabel")
plt.ylabel("ylabel")
plt.title("Histogram Graph Title")
 
plt.show()
 
 
# ------------------------------
# -- Advanced Stats. Analysis
# ------------------------------
 
# --
# -- Draw CDF(cumulative distribution function, 누적분포함수)
# --
print(df_rates[df_rates['Both']>40])
 
def plot_cdf(data, plot_range=None, scale_to=None, nbins=False, **kwargs):
    if not nbins:
        nbins= len(data)
 
    sorted_data = np.array(sorted(data), dtype=np.float64)
    data_range = sorted_data[-1] - sorted_data[0]
    counts, bin_edges = np.histogram(sorted_data, bins=nbins)
    xvalues = bin_edges[1:]
    yvalues = np.cumsum(counts)
 
    if plot_range is None:
        xmin = xvalues[0]
        xmax = xvalues[-1]
    else:
        xmin, xmax = plot_range
 
    # pad the arrays
    xvalues = np.concatenate([[xmin, xvalues[0]], xvalues, [xmax]])
    yvalues = np.concatenate([[0.0, 0.0],        yvalues, [yvalues.max()]])
 
    if scale_to:
        yvalues = yvalues / len(data) * scale_to
    plt.axis([xmin, xmax, 0, yvalues.max()])
 
    return plt.step(xvalues, yvalues, **kwargs)
 
plot_cdf(df_rates['Both'], nbins=50, plot_range=[-5, 70])
 
plt.show()
 
# --
# -- Draw probplot(확률 도표) w/ Scipy
# --
st.probplot(df_rates['Both'], dist='norm', plot=plt);
plt.show()
 
# --
# -- Draw Weibull Distribution(와이블 분포) w/ Scipy
# --
eta = 1.
beta = 1.5
rvweib = st.weibull_min(beta, scale=eta)
results = st.probplot(df_rates['Both'], dist=rvweib, plot=plt)
 
plt.show()
 
# --
# -- Draw weibull_min() w/ Scipy
# --
beta, loc, eta = st.weibull_min.fit(df_rates['Both'], floc=0, scale = 12)
print(beta, loc, eta)
 
df_rates['Both'].hist(bins=30)
np.random.seed(1100)
rvweib = st.weibull_min(beta, scale=eta)
plt.hist(rvweib.rvs(size=len(df_rates.Both)),bins=30, alpha=0.5);
 
plt.show()
 
 
# ------------------------------
# -- Odd the data on df_rates
# ------------------------------
 
# --
# -- Get the new data
# --
df_coords = pd.read_csv('D:\\PyProject\\20170524_acorn1\\country_centroids_primary.csv', sep='\t')
print ("[keys] ! ", df_coords.keys())
print (df_coords.head())
 
print (",,, df_coords.keys", "," * 100, "\n")
 
# --
# -- Add new fields on df_rates
# --
df_rates['Lat'] = ''
df_rates['Lon'] = ''
 
# -- add fields
for i in df_coords.index:
    ind = df_rates.Country.isin([df_coords.SHORT_NAME[i]])
    val = df_coords.loc[i, ['LAT', 'LONG']].values.astype('float')
    df_rates.loc[ind, ['Lat', 'Lon'] ] = list(val)
 
# -- alter type to float
df_rates.loc[df_rates.Lat.isin(['']), ['Lat']] = np.nan
df_rates.loc[df_rates.Lon.isin(['']), ['Lon']] = np.nan
df_rates[['Lat', 'Lon']] = df_rates[['Lat', 'Lon']].astype('float')
 
# -- add DEF field
df_rates['DFE'] = ''
df_rates['DFE'] = abs(df_rates.Lat)
df_rates['DFE'] = df_rates['DFE'].astype('float')
print (df_rates.head())
 
print (",,, add_field_on_df_rates", "," * 100, "\n")
 
 
# ------------------------------
# -- Draw Rectangle, hist w/ Matplotlib.patches
# ------------------------------
 
# --
# -- Draw Rectangle
# --
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(df_rates.DFE, df_rates.Both, '*')
 
trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
 
rect = patches.Rectangle((0,0), width=23.5, height=1,
                         transform=trans, color='yellow',
                         alpha=0.5)
ax.set_xlabel('DFE')
ax.set_ylabel('Both')
ax.add_patch(rect);
 
plt.show()
 
# --
# -- Draw hist
# --
df_rates.DFE.hist(bins=13)
plt.xlabel('DFE')
plt.ylabel('Counts');
 
plt.show()
 
# --
# -- Draw Rectangle include errorbar()
# --
bins = np.arange(23.5, 65+1,10, dtype='float')
bins = np.linspace(23.5, 65,11, dtype='float')
# now group the data into the bins
groups_df_rates = df_rates.groupby(np.digitize(df_rates.DFE, bins))
 
fig = plt.figure()
ax = fig.add_subplot(111)
ax.errorbar(groups_df_rates.mean().DFE,
            groups_df_rates.mean().Both,
            yerr=np.array(groups_df_rates.std().Both),
            marker='.',
            ls='None',
            lw=1.5,
            color='Green',
            ms=1)
ax.plot(df_rates.DFE, df_rates.Both, '.', color='SteelBlue', ms=6)
trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
 
rect = patches.Rectangle((0,0), width=23.5, height=1,
                         transform=trans, color='Yellow',
                         alpha=0.5)
ax.add_patch(rect)
ax.set_xlabel('DFE')
ax.set_ylabel('Both');
 
plt.show()
 
 
# ------------------------------
# -- Save to .CSV
# ------------------------------
df_rates.to_csv("D:\\PyProject\\20170524_acorn1\\df_rates.csv")
 
 
print (";;; saved_to_csv", ";" * 100, "\n")

■ Figure

country_centroids_primary.csv

df_rates.csv

who_suicide_rates.csv