Python Pandas

# Import pandas as pd

import pandas as pd

# Import the cars.csv data: cars cars = pd.read_csv('cars.csv', index_col = 0)

climate_change = pd.read_csv('climate_change.csv' , parse_dates = ['date'], index_col = 'date')

 

# Save as .csv

airline_totals_sorted.to_csv("airline_totals_sorted.csv")

# Create dictionary my_dict with three key:value pairs: my_dict

my_dict = {'country':names, 'drives_right': dr, 'cars_per_cap': cpc}

# Build a DataFrame cars from my_dict: cars

cars = pd.DataFrame(my_dict) 

# Definition of row_labels

row_labels = ['US', 'AUS', 'JPN', 'IN', 'RU', 'MOR', 'EG'] 

# Specify row labels of cars

cars.index = row_labels

# Print out country column as Pandas Series

print(cars['country'])

# Print out country column as Pandas DataFrame

print(cars[['country']])

# Print out DataFrame with country and drives_right columns

print(cars[['country','drives_right']])

# Print out first 3 observations

print(cars[0:3])

# Print out fourth, fifth and sixth observation

print(cars[3:6])

# Print out every 2 observation

print(cars.loc[0::2])

# Print out observations for Australia and Egypt

print(cars.loc[['AUS','EG']])

# Print out drives_right value of Morocco

print(cars.loc['MOR','drives_right']) 

# Print sub-DataFrame

print(cars.loc[['RU', 'MOR'],['country','drives_right']]) 

# Print out drives_right column as Series

print(cars.loc[:,'drives_right'])

# Print out drives_right column as DataFrame

print(cars.loc[:, ['drives_right']])

# Print out cars_per_cap and drives_right as DataFrame

print(cars.loc[:,['cars_per_cap','drives_right']])

 

# Print the head of the sales DataFrame
print(sales.head())
# Print the info about the sales DataFrame
print(sales.info())
# Print the mean of weekly_sales
print(sales["weekly_sales"].mean())
# Print the median of weekly_sales
print(sales["weekly_sales"].median())
# Print the maximum of the date column
print(sales["date"].max())
# Print the minimum of the date column
print(sales["date"].min())

# fill missing values with 0s; sum all rows and cols

print(sales.pivot_table(values="weekly_sales", index="department", columns="type", fill_value=0, margins=True))

 

# Sort temperatures_ind by index values at the city level

print(temperatures_ind.sort_index(level="city"))

# Sort temperatures_ind by country then descending city

print(temperatures_ind.sort_index(level=["country","city"],ascending = [True, False]))

# Sort the index of temperatures_ind

temperatures_srt = temperatures_ind.sort_index()

# A custom IQR function
def iqr(column):
return column.quantile(0.75) - column.quantile(0.25)
# Print IQR of the temperature_c column
print(sales['temperature_c'].agg(iqr))

# Update to print IQR of temperature_c, fuel_price_usd_per_l, & unemployment
print(sales[["temperature_c", "fuel_price_usd_per_l", "unemployment"]].agg(iqr))

# Update to print IQR and median of temperature_c, fuel_price_usd_per_l, & unemployment
print(sales[["temperature_c", "fuel_price_usd_per_l", "unemployment"]].agg([iqr,np.median]))

 

# Subset rows from Pakistan to Russia

print(temperatures_srt.loc["Pakistan":"Russia"])

# Subset rows from India, Hyderabad to Iraq, Baghdad

print(temperatures_srt.loc[("India","Hyderabad"):("Iraq", "Baghdad")])

# Subset columns from date to avg_temp_c

print(temperatures_srt.loc[:, "date":"avg_temp_c"])

# Subset in both directions at once

print(temperatures_srt.loc[("India","Hyderabad"):("Iraq", "Baghdad"), "date":"avg_temp_c"])

 

# Get the total number of avocados sold of each size

nb_sold_by_size = avocados.groupby("size")["nb_sold"].sum()

# select data when type is "conventional"

avocados[avocados["type"] == "conventional"]

# Iterate over rows of cars

for lab, row in cars.iterrows() :

    print(lab)

    print(row)

for lab, row in cars.iterrows() :

    print(lab +": " + str(row['cars_per_cap']))

for lab, row in cars.iterrows() :

    cars.loc[lab, "COUNTRY"] = str.upper(row["country"])

# Use .apply(str.upper) 

for lab, row in cars.iterrows() :

    cars.loc[lab, "COUNTRY"] = row["country"].upper()

# or just use

cars["COUNTRY"] = cars["country"].apply(str.upper)

# Sort sales by date
sales = sales.sort_values("date", ascending=True)

# Get the cumulative sum of weekly_sales, add as cum_weekly_sales col
sales["cum_weekly_sales"] = sales["weekly_sales"].cumsum()

# Get the cumulative max of weekly_sales, add as cum_max_sales col
sales["cum_max_sales"] = sales["weekly_sales"].cummax()

# See the columns you calculated
print(sales[["date", "weekly_sales", "cum_weekly_sales", "cum_max_sales"]])

# Drop duplicate store/type combinations
store_types = sales.drop_duplicates(subset=["store","type"])
print(store_types.head())


# Subset the rows where is_holiday is True and drop duplicate dates
holiday_dates = sales[sales["is_holiday"]].drop_duplicates(subset="date")


# Count the number of stores of each type
store_counts = store_types["type"].value_counts()
print(store_counts)

# Get the proportion of stores of each type
store_props = store_types["type"].value_counts(normalize=True)
print(store_props)

# Count the number of each department number and sort
dept_counts_sorted = store_depts["department"].value_counts(sort=True)
print(dept_counts_sorted)

# Get the proportion of departments of each number and sort
dept_props_sorted = store_depts["department"].value_counts(sort=True, normalize=True)
print(dept_props_sorted)


# Calc total weekly sales
sales_all = sales["weekly_sales"].sum()

# Subset for type A stores, calc total weekly sales
sales_A = sales[sales["type"] == "A"]["weekly_sales"].sum()


# Get proportion for each type
sales_propn_by_type = [sales_A, sales_B, sales_C] / (sales_A+sales_B+sales_C)
print(sales_propn_by_type)


# Group by type; calc total weekly sales
sales_by_type = sales.groupby("type")["weekly_sales"].sum()

# Get proportion for each type
sales_propn_by_type = sales_by_type / sum(sales_by_type)
print(sales_propn_by_type)


# Group by type and is_holiday; calc total weekly sales
sales_by_type_is_holiday = sales.groupby(["type","is_holiday"])["weekly_sales"].sum()
print(sales_by_type_is_holiday)


# For each store type, aggregate unemployment and fuel_price_usd_per_l: get min, max, mean, and median
unemp_fuel_stats = sales.groupby("type")[["unemployment","fuel_price_usd_per_l"]].agg([np.min,np.max,np.mean,np.median])

# Pivot for mean and median weekly_sales for each store type
mean_med_sales_by_type = sales.pivot_table(values="weekly_sales",index="type",aggfunc=[np.mean,np.median])


# Pivot for mean weekly_sales by store type and holiday
mean_sales_by_type_holiday = sales.pivot_table(values="weekly_sales",index="type",columns="is_holiday")


print(temp_by_country_city_vs_year.loc[("Egypt","Cairo"):("India","Delhi","2005":"2010"])


# Get the worldwide mean temp by year
mean_temp_by_year = temp_by_country_city_vs_year.mean(axis="index")

print(mean_temp_by_year.head())
# Filter for the year that had the highest mean temp
print(mean_temp_by_year[mean_temp_by_year==mean_temp_by_year.max()])

# Get the mean temp by city
mean_temp_by_city = temp_by_country_city_vs_year.mean(axis="columns")
print(mean_temp_by_city.head())
# Filter for the city that had the lowest mean temp
print(mean_temp_by_city[mean_temp_by_city==mean_temp_by_city.min()])

 

Python Missing Data

# Check individual values for missing values

print(avocados_2016.isna())

# Check each column for missing values

print(avocados_2016.isna().any())

# Bar plot of missing values by variable

avocados_2016.isna().sum().plot(kind="bar")

# Remove rows with missing values

avocados_complete = avocados_2016.dropna()

# List the columns with missing values

cols_with_missing = ["small_sold", "large_sold", "xl_sold"]

# Create histograms showing the distributions cols_with_missing

avocados_2016[cols_with_missing].hist()

# Fill in missing values with 0

avocados_filled = avocados_2016.fillna(0)

# Create histograms of the filled columns

avocados_filled[cols_with_missing].hist()

Simulations:

def sample(brown, n=1000):

    return pd.dataframe({'vote': np.where(np.random.rand(n) < brown, 'Brown','Green')}) 

dist=pd.dataframe({'Brown': sample(0.50, 1000).vote.value_count(normalize=True)['Brown'], fro i in range(10000)})

dist.Brown.hist(histtype='step',bins=20)

P_value = 100-scipy.stats.percentileofscore(dist.Brown, 0.511)

# Create dummy variables: df_region

df_region = pd.get_dummies(df)

# Create dummy variables with drop_first=True: df_region

df_region = pd.get_dummies(df, drop_first = True)# From previous step

diet.index = pd.to_datetime(diet.index)

# Slice the dataset to keep only 2012

diet2012 = diet['2012']

# Plot 2012 data

diet2012.plot(grid = True)

plt.show()

# Convert the daily data to weekly data

MSFT = MSFT.resample(rule='w', how='last')

# Compute the percentage change of prices

returns = MSFT.pct_change()

# Compute and print the autocorrelation of returns

autocorrelation = returns['Adj Close'].autocorr()

print("The autocorrelation of weekly returns is %4.2f" %(autocorrelation))

# Convert the daily data to annual data

yearly_rates = daily_rates.resample(rule='A', how='last')

# Repeat above for annual data

yearly_diff = yearly_rates.diff()

autocorrelation_yearly = yearly_diff['US10Y'].autocorr()

# Import the acf module and the plot_acf module from statsmodels

from statsmodels.tsa.stattools import acf

from statsmodels.graphics.tsaplots import plot_acf

# Compute the acf array of HRB

acf_array = acf(HRB)

# Plot the acf function

plot_acf(HRB, alpha=1)

plt.show()

plot_acf(returns,alpha=0.05, lags = 20)

Random. Walk

# Generate 500 random steps with mean=0 and standard deviation=1

steps = np.random.normal(loc=0, scale=1, size=500)

# Set first element to 0 so that the first price will be the starting stock price

steps[0]=0

# Simulate stock prices, P with a starting price of 100

P = 100 + np.cumsum(steps)

# Import the adfuller module from statsmodels

from statsmodels.tsa.stattools import adfuller

# Run the ADF test on the price series and print out the results

results = adfuller(AMZN['Adj Close'])

print(results)

# Just print out the p-value

print('The p-value of the test on prices is: ' + str(results[1]))

Statistics : results[0]

# Seasonally adjust quarterly earnings

HRBsa = HRB.diff(4)

# Print the first 10 rows of the seasonally adjusted series

print(HRBsa.head(10))

# import the module for simulating data

from statsmodels.tsa.arima_process import ArmaProcess

# Plot 1: AR parameter = +0.9

plt.subplot(2,1,1)

ar1 = np.array([1, -0.9])

ma1 = np.array([1])

AR_object1 = ArmaProcess(ar1, ma1)

simulated_data_1 = AR_object1.generate_sample(nsample=1000)

plt.plot(simulated_data_1)

# Import the ARMA module from statsmodels

from statsmodels.tsa.arima_model import ARMA

# Fit an AR(1) model to the first simulated data

mod = ARMA(simulated_data_1, order=(1,0))

res = mod.fit()

# Print out summary information on the fit

print(res.summary())

# Print out the estimate for the constant and for phi

print("When the true phi=0.9, the estimate of phi (and the constant) are:")

print(res.params)

res.plot_predict(start=990, end=1010)

res.plot_predict(start=0, end='2022')

plt.legend(fontsize=8)

# Simulate AR(2) with phi1=+0.6, phi2=+0.3

ma = np.array([1])

ar = np.array([1, -0.6, -0.3])

AR_object = ArmaProcess(ar, ma)

simulated_data_2 = AR_object.generate_sample(nsample=5000)

# Plot PACF for AR(2)

plot_pacf(simulated_data_2, lags=20)

plt.show()

 

# Import the module for estimating an ARMA model

from statsmodels.tsa.arima_model import ARMA

# Fit the data to an AR(p) for p = 0,...,6 , and save the BIC

BIC = np.zeros(7)

for p in range(7):

mod = ARMA(simulated_data_2, order=(p,0))

res = mod.fit()

# Save BIC for AR(p)

BIC[p] = res.bic

# Plot the BIC as a function of p

plt.plot(range(1,7), BIC[1:7], marker='o')

plt.xlabel('Order of AR Model')

plt.ylabel('Bayesian Information Criterion')

plt.show()

 

 

# Create a series out of the Country column

countries = so_survey_df['Country']

# Get the counts of each category

country_counts = countries.value_counts()

# Create a mask for only categories that occur less than 10 times

mask = countries.isin(country_counts[country_counts < 10].index)

# Label all other categories as Other

countries[mask] = 'Other'

# Print the updated category counts

print(countries.value_counts())

 

# Create the Paid_Job column filled with zeros

so_survey_df['Paid_Job'] = 0

# Replace all the Paid_Job values where ConvertedSalary is > 0

so_survey_df.loc[so_survey_df.ConvertedSalary > 0, 'Paid_Job'] = 1

 

# Import numpy

import numpy as np

# Specify the boundaries of the bins

bins = [-np.inf, 10000, 50000, 100000, 150000, np.inf]

# Bin labels

labels = ['Very low', 'Low', 'Medium', 'High', 'Very high']

# Bin the continuous variable ConvertedSalary using these boundaries

so_survey_df['boundary_binned'] = pd.cut(so_survey_df['ConvertedSalary'],

bins = bins, labels=labels)

# Replace all non letter characters with a whitespace

speech_df['text_clean'] = speech_df['text'].str.replace('[^a-zA-Z]', ' ')

# Change to lower case

speech_df['text_clean'] = speech_df['text_clean'].str.lower()

 

# Find the length of each text

speech_df['char_cnt'] = speech_df['text_clean'].str.len()

# Count the number of words in each text

speech_df['word_cnt'] = speech_df['text_clean'].str.split().str.len()

# Find the average length of word

speech_df['avg_word_length'] = speech_df['char_cnt'] / speech_df['word_cnt']

 

# Import CountVectorizer

from sklearn.feature_extraction.text import CountVectorizer

# Instantiate CountVectorizer

cv = CountVectorizer(min_df = 0.2, max_df = 0.8)

# Fit the vectorizer

cv.fit(speech_df['text_clean'])

# Print feature names

print(cv.get_feature_names())

 

# Apply the vectorizer

cv_transformed = cv.transform(speech_df['text_clean'])

# Print the full array

cv_array = cv_transformed.toarray()

 

# Create a DataFrame with these features

cv_df = pd.DataFrame(cv_array,

columns=cv.get_feature_names()).add_prefix('Counts_')

# Add the new columns to the original DataFrame

speech_df_new = pd.concat([speech_df, cv_df], axis=1, sort=False)

 

# Import TfidfVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

# Instantiate TfidfVectorizer

tv = TfidfVectorizer(max_features=100, stop_words='english')

# Fit the vectroizer and transform the data

tv_transformed = tv.fit_transform(speech_df['text_clean'])

# Create a DataFrame with these features

tv_df = pd.DataFrame(tv_transformed.toarray(),

columns=tv.get_feature_names()).add_prefix('TFIDF_')

# Isolate the row to be examined

sample_row = tv_df.iloc[0, :]

# Print the top 5 words of the sorted output

print(sample_row.sort_values(ascending=False).head())

 

# Instantiate TfidfVectorizer

tv = TfidfVectorizer(max_features=100, stop_words='english')

# Fit the vectroizer and transform the data

tv_transformed = tv.fit_transform(train_speech_df['text_clean'])

# Transform test data

test_tv_transformed = tv.transform(test_speech_df['text_clean'])

# Create new features for the test set

test_tv_df = pd.DataFrame(test_tv_transformed.toarray(),

columns=tv.get_feature_names()).add_prefix('TFIDF_')

print(test_tv_df.head())

 

# Import CountVectorizer

from sklearn.feature_extraction.text import CountVectorizer

# Instantiate a trigram vectorizer

cv_trigram_vec = CountVectorizer(max_features=100,

stop_words='english',

ngram_range = (3,3))

# Fit and apply trigram vectorizer

cv_trigram = cv_trigram_vec.fit_transform(speech_df['text_clean'])

# Print the trigram features

print(cv_trigram_vec.get_feature_names())

 

# Create a DataFrame of the features

cv_tri_df = pd.DataFrame(cv_trigram.toarray(),

columns=cv_trigram_vec.get_feature_names()).add_prefix('Counts_')

# Print the top 5 words in the sorted output

print(cv_tri_df.sum().sort_values(ascending=False).head())

 

# Drop all rows where Gender is missing

no_gender = so_survey_df.dropna(subset=['Gender'])

# Create a new DataFrame dropping all columns with incomplete rows

no_missing_values_cols = so_survey_df.dropna(how='any', axis=1)

 

 

# Replace the offending characters

so_survey_df['RawSalary'] = so_survey_df['RawSalary'].str.replace('£', '')

# Convert the column to float

so_survey_df['RawSalary'] = so_survey_df['RawSalary'].astype('float')