Python二維數據分析

一.numpy二維數組

1.聲明

import numpy as np

#每一個[]代表一行
ridership = np.array([
    [   0,    0,    2,    5,    0],
    [1478, 3877, 3674, 2328, 2539],
    [1613, 4088, 3991, 6461, 2691],
    [1560, 3392, 3826, 4787, 2613],
    [1608, 4802, 3932, 4477, 2705],
    [1576, 3933, 3909, 4979, 2685],
    [  95,  229,  255,  496,  201],
    [   2,    0,    1,   27,    0],
    [1438, 3785, 3589, 4174, 2215],
    [1342, 4043, 4009, 4665, 3033]
])

2.取值

print ridership[1,3] #取第二行,第四個數
print ridership[1:3,3:5] #取子集
print ridership[1,:]#取第二行
print ridership[0,:]+ridership[1,:]#第一行的值加上第二行的值
print ridership[:,0]+ridership[:,1]#第一列的值加上第二列的值

3.numpy二維數組之間相加

根據線性代數的規則進行相加

a=np.array([[1,2,3],[4,5,6],[7,8,9]])
b=np.array([[1,2,3],[4,5,6],[7,8,9]])
print a+b

4.numpy二維數組的求和

a = np.array([
        [1, 2, 3],
        [4, 5, 6],
        [7, 8, 9]
    ])

print a.sum() #求出數組所有值的和
print a.sum(axis=0) #按照行計算總和
print a.sum(axis=1) #按照列計算總和

例:

1.求出最大車流的車站的平均數

def mean_riders_for_max_station(ridership):
    max_station = ridership[0,:].argmax() #找出最大的車站的下標
    overall_mean = ridership.mean() #求出總體的平均值
    mean_for_max = ridership[:,max_station].mean() #根據最大的車站,找到該下標的所有列,求出平均值
    return (overall_mean,mean_for_max)

2.求出平均車流量的最大和最小的平均值

def min_and_max_riders_per_day(ridership):
    max_station = ridership.mean(axis=0)
    max_daily_ridership = max_station.max()
    min_daily_ridership = max_station.min()
    return (max_daily_ridership,min_daily_ridership)

二.pandas二維數組

1.聲明

import pandas as pd
ridership_df = pd.DataFrame(
    data=[[   0,    0,    2,    5,    0],
          [1478, 3877, 3674, 2328, 2539],
          [1613, 4088, 3991, 6461, 2691],
          [1560, 3392, 3826, 4787, 2613],
          [1608, 4802, 3932, 4477, 2705],
          [1576, 3933, 3909, 4979, 2685],
          [  95,  229,  255,  496,  201],
          [   2,    0,    1,   27,    0],
          [1438, 3785, 3589, 4174, 2215],
          [1342, 4043, 4009, 4665, 3033]],
    index=['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
           '05-06-11', '05-07-11', '05-08-11', '05-09-11', '05-10-11'],
    columns=['R003', 'R004', 'R005', 'R006', 'R007']
)

2.取值

print ridership_df #以表格的方式展示
print ridership_df.iloc[0] #獲取第一行的數據
print ridership_df.loc['05-05-11']#找到該下標所對應行的數據
print ridership_df['R003'] #找出該下標所對應的列的數據
print ridership_df.iloc[1,3] #根據點找出對應的值
print ridership_df[['R003','R005']] #根據多個下標獲取多列的數據

3.行列展示

df_1=pd.DataFrame({'A':[0,1,2],'B':[3,4,5]}) #默認以列的方式展示
print df_1
df_2=pd.DataFrame([[0,1,2],[3,4,5]],columns=['A','B','C']) #以行的方式展示
print df_2

例:

1.求出最大車流的車站的平均數

def mean_riders_for_max_station(ridership):
    max_station = ridership.iloc[0].argmax() #獲取第一行的最大的下邊,即為最繁忙的地鐵站
    overall_mean = ridership.values.mean() #獲取數組中值的所有平均數
    mean_for_max = ridership[max_station].mean() #取出平均該列的的客流量
    return (overall_mean,mean_for_max)

2.相關性

概念:相關性的值(-1,+1),如果越接近+1表示兩個量程正相關,越接近-1表示程負相關,接近0表示越無關

subway_df = pd.read_csv('nyc-subway-weather.csv') #下載區有文件
def correlation(x,y):
    #該函數體現了表格中兩個變量之間的相關性
    std_x = (x-x.mean())/x.std(ddof=0)
    std_y = (y-y.mean())/y.std(ddof=0)
    return (std_x*std_y).mean()

entries = subway_df['ENTRIESn_hourly'] #獲取出對應列的值
cum_entries = subway_df['ENTRIESn']
rain = subway_df['meanprecipi']
temp = subway_df['meantempi']

#找出兩個變量之間的關聯度
print correlation(entries,rain)
print correlation(entries,temp)
print correlation(rain,temp)
print correlation(entries,cum_entries)

3.DataFrame相關操作

符合線性代數的計算規則,如果兩個DataFrame的值不同,就用NaN值填充

df1=pd.DataFrame({'a':[1,2,3],'b':[4,5,6],'c':[7,8,9]})
df2=pd.DataFrame({'a':[10,20,30],'b':[40,50,60],'c':[70,80,90]})
print df1+df2

df1 = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
df2 = pd.DataFrame({'d': [10, 20, 30], 'c': [40, 50, 60], 'b': [70, 80, 90]})
print df1 + df2

df1=pd.DataFrame({'a':[1,2,3],'b':[4,5,6],'c':[7,8,9]},index=['row1','row2','row3'])
df2=pd.DataFrame({'a':[10,20,30],'b':[40,50,60],'c':[70,80,90]},index=['row1','row2','row3'])
print df1+df2

例:

1.求出每小時的地鐵的流量

entries_and_exits = pd.DataFrame({
    'ENTRIESn': [3144312, 3144335, 3144353, 3144424, 3144594,
                 3144808, 3144895, 3144905, 3144941, 3145094],
    'EXITSn': [1088151, 1088159, 1088177, 1088231, 1088275,
               1088317, 1088328, 1088331, 1088420, 1088753]
})
#獲取每小時的進出地鐵的人數
def get_hourly_entries_and_exits(entries_and_exits):
    return entries_and_exits-entries_and_exits.shift(1)

4.applymap方法,將DataFrame的所有值通過自定義方法得以修改

df = pd.DataFrame({
    'a': [1, 2, 3],
    'b': [10, 20, 30],
    'c': [5, 10, 15]
})

def add_one(x):
    return x + 1


print df.applymap(add_one)

例:

1.將學生的成績轉換為等級(A,B,C,D,F)

grades_df = pd.DataFrame(
    data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
          'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
    index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio', 
           'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)
def convert_grade(grade):
    if grade>=90:
        return 'A'
    elif grade>=80:
        return 'B'
    elif grade>=70:
        return 'C'
    elif grade>=60:
        return 'D'
    else:
        return 'F'
def convert_grades(grades):
    return grades.applymap(convert_grade)

2.計算標准差

def standardize_column(df):
    return (df-df.mean())/df.std(ddof=0) #ddof表示使用貝塞爾教正參數

def standardize(df):
    return df.apply(standardize_column)
print standardize(grades_df)

3.獲取數組中第二大的值

df = pd.DataFrame({
    'a': [4, 5, 3, 1, 2],
    'b': [20, 10, 40, 50, 30],
    'c': [25, 20, 5, 15, 10]
})

def second_largest_column(column):
    #將數據以降序進行排列,下標為1的就是第二大的數
    sorted_column = column.sort_values(ascending=False)
    return sorted_column.iloc[1]

def senond_large(df):
    return df.apply(second_largest_column)

5.DataFrame和Seriers操作

s=pd.Series([1,2,3,4])
df=pd.DataFrame({0:[10,20,30,40],1:[50,60,70,80],2:[90,100,110,120],3:[130,140,150,160]})
print df+s #每行對應數值相加

df = pd.DataFrame({0: [10], 1: [20], 2: [30], 3: [40]})
print df+s #行相加

df = pd.DataFrame({0: [10, 20, 30, 40]})
print df+s #列相加

s = pd.Series([1, 2, 3, 4], index=['a', 'b', 'c', 'd'])
df = pd.DataFrame({
    'a': [10, 20, 30, 40],
    'b': [50, 60, 70, 80],
    'c': [90, 100, 110, 120],
    'd': [130, 140, 150, 160]
})
print df+s #默認進行行相加
print df.add(s,axis='index') #列相加,此處不匹配,顯示NaN值
print df.add(s,axis='columns') #指定進行行相加

例:

1.計算學生的標准偏差

grades_df = pd.DataFrame(
    data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
          'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
    index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio', 
           'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)

#計算每個學生的標准偏差
def standardize(df):
    return (df-df.mean())/df.std(ddof=0)

def standardize_row(df):
    #1.計算每個學生考試成績和平均成績的差
    #2.再計算每個學生的樣本偏差
    mean_diffs = df.sub(df.mean(axis='columns'),axis='index')
    return mean_diffs.div(df.std(axis='columns',ddof=0),axis='index')

6.groupby分組

values = np.array([1, 3, 2, 4, 1, 6, 4])
example_df = pd.DataFrame({
    'value': values,
    'even': values % 2 == 0,
    'above_three': values > 3 
}, index=['a', 'b', 'c', 'd', 'e', 'f', 'g'])

print example_df

#根據單列進行分組
grouped_data = example_df.groupby('even')
print grouped_data.groups
#根據多列進行分組
grouped_data = example_df.groupby(['even','above_three'])
print grouped_data.groups

grouped_data = example_df.groupby('even')
print grouped_data.sum() #分組后求和
print grouped_data.sum()['value'] #先求和再根據value分組
print grouped_data['value'].sum() #繼續根據value分組在求和

例:

1.計算value的標准偏差

def standardize(xs):
    return (xs-xs.mean())/xs.std(ddof=0)

#根據even字段分組
grouped_data=example_df.groupby('even')

#根據value在分組然后計算標准差
print grouped_data['value'].apply(standardize)

2.畫出車站每周每小時的使用的平均值

%pylab inline
import seaborn as sns
subway_df = pd.read_csv('nyc-subway-weather.csv')
#根據day_week分組,然后獲取平均值,最后獲取ENTRIESn_hourly列的值
ridership_by_day = subway_df.groupby('day_week').mean()['ENTRIESn_hourly']
ridership_by_day.plot()

3.獲取每個地鐵站每個小時的流量

ridership_df = pd.DataFrame({
    'UNIT': ['R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051'],
    'TIMEn': ['00:00:00', '02:00:00', '04:00:00', '06:00:00', '08:00:00', '10:00:00', '12:00:00', '14:00:00', '16:00:00'],
    'ENTRIESn': [3144312, 8936644, 3144335, 8936658, 3144353, 8936687, 3144424, 8936819, 3144594],
    'EXITSn': [1088151, 13755385,  1088159, 13755393,  1088177, 13755598, 1088231, 13756191,  1088275]
})
def hour_by_group(entries_and_exits):
    return entries_and_exits-entries_and_exits.shift(1)
#獲取每個車站的每小時的進出口
def get_hourly_entries_and_exits(entries_and_exits):
    #根據UNIT字段分組,然后,獲取相應的列,最后調用自定義的方法得出結論
    return entries_and_exits.groupby('UNIT')['ENTRIESn','EXITSn'].apply(hour_by_group)

7.merge組合,將兩個結果集根據某些字段進行組合,整合為一個結果集

subway_df = pd.DataFrame({
    'UNIT': ['R003', 'R003', 'R003', 'R003', 'R003', 'R004', 'R004', 'R004',
             'R004', 'R004'],
    'DATEn': ['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
              '05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11'],
    'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    'ENTRIESn': [ 4388333,  4388348,  4389885,  4391507,  4393043, 14656120,
                 14656174, 14660126, 14664247, 14668301],
    'EXITSn': [ 2911002,  2911036,  2912127,  2913223,  2914284, 14451774,
               14451851, 14454734, 14457780, 14460818],
    'latitude': [ 40.689945,  40.689945,  40.689945,  40.689945,  40.689945,
                  40.69132 ,  40.69132 ,  40.69132 ,  40.69132 ,  40.69132 ],
    'longitude': [-73.872564, -73.872564, -73.872564, -73.872564, -73.872564,
                  -73.867135, -73.867135, -73.867135, -73.867135, -73.867135]
})

weather_df = pd.DataFrame({
    'DATEn': ['05-01-11', '05-01-11', '05-02-11', '05-02-11', '05-03-11',
              '05-03-11', '05-04-11', '05-04-11', '05-05-11', '05-05-11'],
    'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    'latitude': [ 40.689945,  40.69132 ,  40.689945,  40.69132 ,  40.689945,
                  40.69132 ,  40.689945,  40.69132 ,  40.689945,  40.69132 ],
    'longitude': [-73.872564, -73.867135, -73.872564, -73.867135, -73.872564,
                  -73.867135, -73.872564, -73.867135, -73.872564, -73.867135],
    'pressurei': [ 30.24,  30.24,  30.32,  30.32,  30.14,  30.14,  29.98,  29.98,
                   30.01,  30.01],
    'fog': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    'rain': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    'tempi': [ 52. ,  52. ,  48.9,  48.9,  54. ,  54. ,  57.2,  57.2,  48.9,  48.9],
    'wspdi': [  8.1,   8.1,   6.9,   6.9,   3.5,   3.5,  15. ,  15. ,  15. ,  15. ]
})

#將相關聯的量組合
def combine_dfs(subway_df,weather_df):
    return subway_df.merge(weather_df,on=['DATEn','hour','latitude','longitude'],how='inner')

#如果兩個列不同,則要通過on_left和on_right來匹配參數

例:

1.做出地鐵站位置的散點圖,通過點的大小展示哪里的車站人流最高

%pylab inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
subway_df = pd.read_csv('nyc-subway-weather.csv')
#根據經度和緯度分組,求出平均數,注意as_index會將字段本身不作為索引,避免出錯
data_for_location = subway_df.groupby(['latitude','longitude'],as_index=False).mean()
#求出每小時的標准偏差,作為圖片大小
scaled_entries = 
data_for_location['ENTRIESn_hourly']/data_for_location['ENTRIESn_hourly'].std(ddof=0)
#根據緯度為x軸,經度為y軸,s的教正系數,做出散點圖
plt.scatter(data_for_location['latitude'],data_for_location['longitude'],s=scaled_entries)