R programming
Lab 6_Model Evaluation.docx MIS 545 Lab 6: Model Evaluation 1 Overview In this lab, we will examine the performance of prediction on two data sets, which can be found under lab 6 module on D2L. Save them in your working directory. 1. adult.csv: This dataset contains census data about more than 48,000 individuals. Try to predict whether an individual’s income exceeds $50K/yr based on census data, such as age, work class, education, race, sex, marital status, country etc. You can find the detail about the dataset at: https://archive.ics.uci.edu/ml/datasets/Adult 2. titanic.csv: This dataset contains variables like class, age, and sex, to figure out if a person survived the wreck of titanic. It has been used in previous lectures. 2 Packages For lab 6, we will use 2 packages to manipulate data. C50: This model extends the C4.5 classification algorithms described in Quinlan (1992). The details of the extensions are largely undocumented. The model can take the form of a full decision tree. pROC: Tools for visualizing, smoothing, and comparing receiver operating characteristic (ROC curves). # Install packages install.packages("C50") install.packages("pROC") library(C50) library(pROC) 3 Precision and Recall First, use setwd() to assign your working directory. Save adult.csv under the directory. Then load adult dataset into the memory, in which question mark stands for missing value. Due to the built-in function of C50 package, we don't have to preprocess missing value. # Read in csv file groceries.csv. adult <- read.csv("adult.csv",="" na.strings='?' )="" split="" data="" for="" training="" and="" testing="" #="" partition="" dataset="" for="" training="" (80%)="" and="" testing="" (20%)="" size=""><- floor(0.8="" *="" nrow(adult))="" ###="" randomly="" decide="" which="" ones="" for="" training="" training_index=""><- sample(nrow(adult),="" size="size," replace="FALSE)" train=""><- adult[training_index,]="" test=""><- adult[-training_index,]="" ###="" names="" of="" variables="" that="" used="" for="" prediction="" var_names=""><- names(adult)[-15]="" fit="" decision="" tree="" model.="" you="" can="" find="" a="" ranked="" list="" of="" attributes="" in="" term="" of="" usage="" via="" method="" summary(dt).="" #="" fit="" the="" model="" dt=""><- c5.0(x="train[," var_names],="" y="train$if_above_50K)" #="" see="" the="" summary="" of="" model="" summary(dt)="" ###="" now,="" validate="" test="" ##="" predict()="" method="" returns="" a="" vector="" of="" result="" dt_pred=""><- predict(dt,="" newdata="test)" ###="" merger="" dt_prediction="" value="" to="" test="" dataset="" dt_evaluation=""><- cbind(test,="" dt_pred)="" have="" a="" simple="" feel="" of="" prediction="" ###="" compare="" dt_prediction="" result="" to="" actual="" value="" dt_evaluation$correct=""><- ifelse(dt_evaluation$if_above_50k="=" dt_evaluation$dt_pred,="" 1,="" 0)="" ###="" accuracy="" rate="" sum(dt_evaluation$correct)="" nrow(dt_evaluation)="" ###="" confusion="" matrix="" table(dt_evaluation$if_above_50k,="" dt_evaluation$dt_pred)="" ##="" no="" yes="" ##="" no="" 4608="" 271="" ##="" yes="" 545="" 1089="" in="" general,="" we="" have="" four="" metrics="" to="" evaluate="" prediction.="" tpr,="" tnr,="" fpr,="" and="" fnr.="" ###="" true="" positive="" rate="" (sensitivity)="" tpr="TP" p="" ###="count" of="" true="" positive="" dt_prediction="" divided="" by="" total="" positive="" truth="" tpr=""><- sum(dt_evaluation$dt_pred="=" 'yes'="" &="" dt_evaluation$if_above_50k="=" 'yes')="" sum(dt_evaluation$if_above_50k="=" 'yes')="" ###="" true="" negative="" rate="" (specificity)="" tnr="TN" n="" ###="count" of="" true="" negative="" dt_prediction="" divided="" by="" total="" negative="" truth="" tnr=""><- sum(dt_evaluation$dt_pred="=" 'no'="" &="" dt_evaluation$if_above_50k="=" 'no')="" sum(dt_evaluation$if_above_50k="=" 'no')="" ###="" false="" positive="" rate="" (1="" -="" spec)="" fpr="FP" n="" ###="count" of="" false="" positive="" dt_prediction="" divided="" by="" total="" negative="" truth="" ###="sum(dt_evaluation$dt_pred" =='yes' &="" dt_evaluation$if_above_50k="=" 'no'="" )/="" ###="" sum(dt_evaluation$if_above_50k="=" 'no')="" fpr=""><- 1="" -="" tnr="" ###="" false="" negative="" rate="" fnr="" fnr="FN" p="" ###="count" of="" false="" negative="" dt_prediction="" divided="" by="" total="" positive="" truth="" ###="sum(dt_evaluation$dt_pred" =='no' &="" dt_evaluation$if_above_50k="=" 'yes'="" )/="" ###="" sum(dt_evaluation$if_above_50k="=" 'yes')="" fnr=""><- 1="" -="" tpr="" precision="" and="" recall="" are="" widely="" used="" to="" evaluate="" prediction="" performance.="" ###="" dt_precision="" equals="" ###="number" of="" true="" positive="" dt_prediction="" total="" positive="" dt_prediction="" dt_precision=""><- sum(dt_evaluation$if_above_50k="=" 'yes'="" &="" dt_evaluation$dt_pred="=" 'yes')="" sum(dt_evaluation$dt_pred="=" 'yes')="" ###="" dt_recall="" equals="TPR" ###="true" positive="" dt_prediction="" total="" true="" positive="" dt_recall=""><- sum(dt_evaluation$if_above_50k="=" 'yes'="" &="" dt_evaluation$dt_pred="=" 'yes')="" sum(dt_evaluation$if_above_50k="=" 'yes')="" f="" score="" is="" a="" metric="" that="" combines="" precision="" and="" recall="" is="" the harmonic="" mean of="" precision="" and="" recall.="" in="" some="" cases,="" we="" have="" to="" adjust="" weight="" of="" precision="" or="" recall="" due="" to="" domain="" knowledge.="" ###="" f="" measure="" f=""><- 2="" *="" dt_precision="" *="" dt_recall="" (dt_precision="" +="" dt_recall)="" 4="" roc="" curve:="" receiver="" operating="" characteristic="" curve="" load="" the="" second="" dataset,="" titanic.csv.="" partition="" data="" into="" training="" and="" testing="" as="" we="" did="" above.="" titanic=""><- read.csv("titanic.csv")="" ###="" partition="" dataset="" for="" training="" (80%)="" and="" testing="" (20%)="" size=""><- floor(0.8="" *="" nrow(titanic))="" ###="" randomly="" decide="" which="" ones="" for="" training="" training_index=""><- sample(nrow(titanic),="" size="size," replace="FALSE)" train=""><- titanic[training_index,]="" test=""><- titanic[-training_index,]="" fit="" logistic="" regression.="" note="" parameter="" type="response" in="" predict="" method.="" it="" returns="" risk="" rate="" instead="" of="" classification.="" ###="" fitting="" regression="" model="" reg=""><- glm(survive="" ~="" .="" ,="" data="train," family="binomial()" )="" ###="" model="" detail="" summary(reg)="" ###="" validate="" test="" dataset="" evaluation=""><- test="" evaluation$prob=""><- predict(reg,="" newdata="evaluation," type="response" )="" see="" the="" improvement="" compared="" to="" baseline="" in="" dataset="" #="" baseline="32%" count_survive=""><- nrow(subset(titanic,="" titanic$survive="=" "yes")="" )="" baseline=""><- count_survive="" nrow(titanic)="" baseline="" ##="" 0.323035="" plot="" roc="" curve="" note="" the="" auc="" is="" 0.7686,="" significantly="" higher="" than="" average="" threshold.="" since="" training="" set="" and="" testing="" set="" are="" randomly="" sampled,="" this="" number="" may="" be="" different="" on="" your="" computer.="" #="" feed="" sensitivity="" &="" specificity="" to="" roc()="" g=""><- roc(evaluation$survive ~ evaluation$prob, data = evaluation) # roc curve plot(g) ## area under the curve: 0.7686 2 adult.csv age,workclass,fnlwgt,education,education-num,marital_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,if_above_50k 39,state-gov,77516,bachelors,13,never-married,adm-clerical,not-in-family,white,male,2174,0,40,united-states,no 50,self-emp-not-inc,83311,bachelors,13,married-civ-spouse,exec-managerial,husband,white,male,0,0,13,united-states,no 38,private,215646,hs-grad,9,divorced,handlers-cleaners,not-in-family,white,male,0,0,40,united-states,no 53,private,234721,11th,7,married-civ-spouse,handlers-cleaners,husband,black,male,0,0,40,united-states,no 28,private,338409,bachelors,13,married-civ-spouse,prof-specialty,wife,black,female,0,0,40,cuba,no 37,private,284582,masters,14,married-civ-spouse,exec-managerial,wife,white,female,0,0,40,united-states,no 49,private,160187,9th,5,married-spouse-absent,other-service,not-in-family,black,female,0,0,16,jamaica,no 52,self-emp-not-inc,209642,hs-grad,9,married-civ-spouse,exec-managerial,husband,white,male,0,0,45,united-states,yes 31,private,45781,masters,14,never-married,prof-specialty,not-in-family,white,female,14084,0,50,united-states,yes 42,private,159449,bachelors,13,married-civ-spouse,exec-managerial,husband,white,male,5178,0,40,united-states,yes 37,private,280464,some-college,10,married-civ-spouse,exec-managerial,husband,black,male,0,0,80,united-states,yes 30,state-gov,141297,bachelors,13,married-civ-spouse,prof-specialty,husband,asian-pac-islander,male,0,0,40,india,yes 23,private,122272,bachelors,13,never-married,adm-clerical,own-child,white,female,0,0,30,united-states,no 32,private,205019,assoc-acdm,12,never-married,sales,not-in-family,black,male,0,0,50,united-states,no 40,private,121772,assoc-voc,11,married-civ-spouse,craft-repair,husband,asian-pac-islander,male,0,0,40,?,yes 34,private,245487,7th-8th,4,married-civ-spouse,transport-moving,husband,amer-indian-eskimo,male,0,0,45,mexico,no 25,self-emp-not-inc,176756,hs-grad,9,never-married,farming-fishing,own-child,white,male,0,0,35,united-states,no 32,private,186824,hs-grad,9,never-married,machine-op-inspct,unmarried,white,male,0,0,40,united-states,no 38,private,28887,11th,7,married-civ-spouse,sales,husband,white,male,0,0,50,united-states,no 43,self-emp-not-inc,292175,masters,14,divorced,exec-managerial,unmarried,white,female,0,0,45,united-states,yes 40,private,193524,doctorate,16,married-civ-spouse,prof-specialty,husband,white,male,0,0,60,united-states,yes 54,private,302146,hs-grad,9,separated,other-service,unmarried,black,female,0,0,20,united-states,no 35,federal-gov,76845,9th,5,married-civ-spouse,farming-fishing,husband,black,male,0,0,40,united-states,no 43,private,117037,11th,7,married-civ-spouse,transport-moving,husband,white,male,0,2042,40,united-states,no 59,private,109015,hs-grad,9,divorced,tech-support,unmarried,white,female,0,0,40,united-states,no 56,local-gov,216851,bachelors,13,married-civ-spouse,tech-support,husband,white,male,0,0,40,united-states,yes 19,private,168294,hs-grad,9,never-married,craft-repair,own-child,white,male,0,0,40,united-states,no 54,?,180211,some-college,10,married-civ-spouse,?,husband,asian-pac-islander,male,0,0,60,south,yes 39,private,367260,hs-grad,9,divorced,exec-managerial,not-in-family,white,male,0,0,80,united-states,no 49,private,193366,hs-grad,9,married-civ-spouse,craft-repair,husband,white,male,0,0,40,united-states,no 23,local-gov,190709,assoc-acdm,12,never-married,protective-serv,not-in-family,white,male,0,0,52,united-states,no 20,private,266015,some-college,10,never-married,sales,own-child,black,male,0,0,44,united-states,no 45,private,386940,bachelors,13,divorced,exec-managerial,own-child,white,male,0,1408,40,united-states,no 30,federal-gov,59951,some-college,10,married-civ-spouse,adm-clerical,own-child,white,male,0,0,40,united-states,no 22,state-gov,311512,some-college,10,married-civ-spouse,other-service,husband,black,male,0,0,15,united-states,no 48,private,242406,11th,7,never-married,machine-op-inspct,unmarried,white,male,0,0,40,puerto-rico,no 21,private,197200,some-college,10,never-married,machine-op-inspct,own-child,white,male,0,0,40,united-states,no 19,private,544091,hs-grad,9,married-af-spouse,adm-clerical,wife,white,female,0,0,25,united-states,no 31,private,84154,some-college,10,married-civ-spouse,sales,husband,white,male,0,0,38,?,yes 48,self-emp-not-inc,265477,assoc-acdm,12,married-civ-spouse,prof-specialty,husband,white,male,0,0,40,united-states,no 31,private,507875,9th,5,married-civ-spouse,machine-op-inspct,husband,white,male,0,0,43,united-states,no 53,self-emp-not-inc,88506,bachelors,13,married-civ-spouse,prof-specialty,husband,white,male,0,0,40,united-states,no 24,private,172987,bachelors,13,married-civ-spouse,tech-support,husband,white,male,0,0,50,united-states,no 49,private,94638,hs-grad,9,separated,adm-clerical,unmarried,white,female,0,0,40,united-states,no 25,private,289980,hs-grad,9,never-married,handlers-cleaners,not-in-family,white,male,0,0,35,united-states,no 57,federal-gov,337895,bachelors,13,married-civ-spouse,prof-specialty,husband,black,male,0,0,40,united-states,yes 53,private,144361,hs-grad,9,married-civ-spouse,machine-op-inspct,husband,white,male,0,0,38,united-states,no 44,private,128354,masters,14,divorced,exec-managerial,unmarried,white,female,0,0,40,united-states,no 41,state-gov,101603,assoc-voc,11,married-civ-spouse,craft-repair,husband,white,male,0,0,40,united-states,no 29,private,271466,assoc-voc,11,never-married,prof-specialty,not-in-family,white,male,0,0,43,united-states,no 25,private,32275,some-college,10,married-civ-spouse,exec-managerial,wife,other,female,0,0,40,united-states,no 18,private,226956,hs-grad,9,never-married,other-service,own-child,white,female,0,0,30,?,no 47,private,51835,prof-school,15,married-civ-spouse,prof-specialty,wife,white,female,0,1902,60,honduras,yes 50,federal-gov,251585,bachelors,13,divorced,exec-managerial,not-in-family,white,male,0,0,55,united-states,yes 47,self-emp-inc,109832,hs-grad,9,divorced,exec-managerial,not-in-family,white,male,0,0,60,united-states,no 43,private,237993,some-college,10,married-civ-spouse,tech-support,husband,white,male,0,0,40,united-states,yes 46,private,216666 roc(evaluation$survive="" ~="" evaluation$prob,="" data="evaluation)" #="" roc="" curve="" plot(g)="" ##="" area="" under="" the="" curve:="" 0.7686="" 2="" adult.csv="" age,workclass,fnlwgt,education,education-num,marital_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,if_above_50k="" 39,state-gov,77516,bachelors,13,never-married,adm-clerical,not-in-family,white,male,2174,0,40,united-states,no="" 50,self-emp-not-inc,83311,bachelors,13,married-civ-spouse,exec-managerial,husband,white,male,0,0,13,united-states,no="" 38,private,215646,hs-grad,9,divorced,handlers-cleaners,not-in-family,white,male,0,0,40,united-states,no="" 53,private,234721,11th,7,married-civ-spouse,handlers-cleaners,husband,black,male,0,0,40,united-states,no="" 28,private,338409,bachelors,13,married-civ-spouse,prof-specialty,wife,black,female,0,0,40,cuba,no="" 37,private,284582,masters,14,married-civ-spouse,exec-managerial,wife,white,female,0,0,40,united-states,no="" 49,private,160187,9th,5,married-spouse-absent,other-service,not-in-family,black,female,0,0,16,jamaica,no="" 52,self-emp-not-inc,209642,hs-grad,9,married-civ-spouse,exec-managerial,husband,white,male,0,0,45,united-states,yes="" 31,private,45781,masters,14,never-married,prof-specialty,not-in-family,white,female,14084,0,50,united-states,yes="" 42,private,159449,bachelors,13,married-civ-spouse,exec-managerial,husband,white,male,5178,0,40,united-states,yes="" 37,private,280464,some-college,10,married-civ-spouse,exec-managerial,husband,black,male,0,0,80,united-states,yes="" 30,state-gov,141297,bachelors,13,married-civ-spouse,prof-specialty,husband,asian-pac-islander,male,0,0,40,india,yes="" 23,private,122272,bachelors,13,never-married,adm-clerical,own-child,white,female,0,0,30,united-states,no="" 32,private,205019,assoc-acdm,12,never-married,sales,not-in-family,black,male,0,0,50,united-states,no="" 40,private,121772,assoc-voc,11,married-civ-spouse,craft-repair,husband,asian-pac-islander,male,0,0,40,?,yes="" 34,private,245487,7th-8th,4,married-civ-spouse,transport-moving,husband,amer-indian-eskimo,male,0,0,45,mexico,no="" 25,self-emp-not-inc,176756,hs-grad,9,never-married,farming-fishing,own-child,white,male,0,0,35,united-states,no="" 32,private,186824,hs-grad,9,never-married,machine-op-inspct,unmarried,white,male,0,0,40,united-states,no="" 38,private,28887,11th,7,married-civ-spouse,sales,husband,white,male,0,0,50,united-states,no="" 43,self-emp-not-inc,292175,masters,14,divorced,exec-managerial,unmarried,white,female,0,0,45,united-states,yes="" 40,private,193524,doctorate,16,married-civ-spouse,prof-specialty,husband,white,male,0,0,60,united-states,yes="" 54,private,302146,hs-grad,9,separated,other-service,unmarried,black,female,0,0,20,united-states,no="" 35,federal-gov,76845,9th,5,married-civ-spouse,farming-fishing,husband,black,male,0,0,40,united-states,no="" 43,private,117037,11th,7,married-civ-spouse,transport-moving,husband,white,male,0,2042,40,united-states,no="" 59,private,109015,hs-grad,9,divorced,tech-support,unmarried,white,female,0,0,40,united-states,no="" 56,local-gov,216851,bachelors,13,married-civ-spouse,tech-support,husband,white,male,0,0,40,united-states,yes="" 19,private,168294,hs-grad,9,never-married,craft-repair,own-child,white,male,0,0,40,united-states,no="" 54,?,180211,some-college,10,married-civ-spouse,?,husband,asian-pac-islander,male,0,0,60,south,yes="" 39,private,367260,hs-grad,9,divorced,exec-managerial,not-in-family,white,male,0,0,80,united-states,no="" 49,private,193366,hs-grad,9,married-civ-spouse,craft-repair,husband,white,male,0,0,40,united-states,no="" 23,local-gov,190709,assoc-acdm,12,never-married,protective-serv,not-in-family,white,male,0,0,52,united-states,no="" 20,private,266015,some-college,10,never-married,sales,own-child,black,male,0,0,44,united-states,no="" 45,private,386940,bachelors,13,divorced,exec-managerial,own-child,white,male,0,1408,40,united-states,no="" 30,federal-gov,59951,some-college,10,married-civ-spouse,adm-clerical,own-child,white,male,0,0,40,united-states,no="" 22,state-gov,311512,some-college,10,married-civ-spouse,other-service,husband,black,male,0,0,15,united-states,no="" 48,private,242406,11th,7,never-married,machine-op-inspct,unmarried,white,male,0,0,40,puerto-rico,no="" 21,private,197200,some-college,10,never-married,machine-op-inspct,own-child,white,male,0,0,40,united-states,no="" 19,private,544091,hs-grad,9,married-af-spouse,adm-clerical,wife,white,female,0,0,25,united-states,no="" 31,private,84154,some-college,10,married-civ-spouse,sales,husband,white,male,0,0,38,?,yes="" 48,self-emp-not-inc,265477,assoc-acdm,12,married-civ-spouse,prof-specialty,husband,white,male,0,0,40,united-states,no="" 31,private,507875,9th,5,married-civ-spouse,machine-op-inspct,husband,white,male,0,0,43,united-states,no="" 53,self-emp-not-inc,88506,bachelors,13,married-civ-spouse,prof-specialty,husband,white,male,0,0,40,united-states,no="" 24,private,172987,bachelors,13,married-civ-spouse,tech-support,husband,white,male,0,0,50,united-states,no="" 49,private,94638,hs-grad,9,separated,adm-clerical,unmarried,white,female,0,0,40,united-states,no="" 25,private,289980,hs-grad,9,never-married,handlers-cleaners,not-in-family,white,male,0,0,35,united-states,no="" 57,federal-gov,337895,bachelors,13,married-civ-spouse,prof-specialty,husband,black,male,0,0,40,united-states,yes="" 53,private,144361,hs-grad,9,married-civ-spouse,machine-op-inspct,husband,white,male,0,0,38,united-states,no="" 44,private,128354,masters,14,divorced,exec-managerial,unmarried,white,female,0,0,40,united-states,no="" 41,state-gov,101603,assoc-voc,11,married-civ-spouse,craft-repair,husband,white,male,0,0,40,united-states,no="" 29,private,271466,assoc-voc,11,never-married,prof-specialty,not-in-family,white,male,0,0,43,united-states,no="" 25,private,32275,some-college,10,married-civ-spouse,exec-managerial,wife,other,female,0,0,40,united-states,no="" 18,private,226956,hs-grad,9,never-married,other-service,own-child,white,female,0,0,30,?,no="" 47,private,51835,prof-school,15,married-civ-spouse,prof-specialty,wife,white,female,0,1902,60,honduras,yes="" 50,federal-gov,251585,bachelors,13,divorced,exec-managerial,not-in-family,white,male,0,0,55,united-states,yes="" 47,self-emp-inc,109832,hs-grad,9,divorced,exec-managerial,not-in-family,white,male,0,0,60,united-states,no="" 43,private,237993,some-college,10,married-civ-spouse,tech-support,husband,white,male,0,0,40,united-states,yes="">