DATA SCIENCE WITH R LAB

# 1. Compute Mean, Median, Mode, Variance, and Standard Deviation
df <- data.frame(
  EMPID = c(101, 102, 103, 104, 105,
            106, 107, 108, 109, 110),
  EMPNAME = c("Alice", "Bob", "Charlie", "David", "Eva",
              "Frank", "Grace", "Hannah", "Ian", "Julia"),
  SALARY = c(50000, 60000, 55000, 70000, 65000,
             48000, 72000, 58000, 69000, 75000),
  STARTDATE = as.Date(c("2020-01-15", "2019-03-10",
                        "2021-06-25", "2018-07-01",
                        "2020-09-12", "2021-11-05",
                        "2017-04-20", "2022-02-14",
                        "2019-12-30", "2016-05-18"))
)

# Mean
mean_salary <- mean(df$SALARY)

# Median
median_salary <- median(df$SALARY)

# Mode function (R doesn't have built-in mode for stats)
getmode <- function(v) {
  uniqv <- unique(v)
  uniqv[which.max(tabulate(match(v, uniqv)))]
}
mode_salary <- getmode(df$SALARY)

# Variance
var_salary <- var(df$SALARY)

# Standard Deviation
sd_salary <- sd(df$SALARY)

# Print all measures
cat("Mean Salary:", mean_salary, "\n")
cat("Median Salary:", median_salary, "\n")
cat("Mode Salary:", mode_salary, "\n")
cat("Variance Salary:", var_salary, "\n")
cat("SD Salary:", sd_salary, "\n")


# 2. Visualize Binomial, Normal, and Poisson Distributions
library(dplyr)
library(ggplot2)

df <- data.frame(
  Experience = c(1,2,3,4,5,6,7,8,9,10),
  SALARY = c(25000,30000,35000,42000,48000,
             52000,58000,65000,72000,80000)
)

# Prepare a tidy data frame for the distributions
mean_salary <- mean(df$SALARY)
sd_salary <- sd(df$SALARY)

n_trials <- 20
p_success <- mean(df$SALARY > 60000)

binom_df <- tibble::tibble(
  x = 0:n_trials,
  prob = dbinom(0:n_trials, size = n_trials, prob = p_success)
)

norm_df <- tibble::tibble(
  x = seq(min(df$SALARY) - 5000, max(df$SALARY) + 5000, length.out = 500),
  density = dnorm(x, mean = mean_salary, sd = sd_salary)
)

lambda_pois <- mean(df$SALARY / 1000)
pois_df <- tibble::tibble(
  x = 0:ceiling(lambda_pois + 10),
  prob = dpois(0:ceiling(lambda_pois + 10), lambda = lambda_pois)
)

# Binomial plot
ggplot(binom_df, aes(x = x, y = prob)) +
  geom_col(fill = "skyblue", color = "black") +
  labs(title = "Binomial Distribution",
       x = "Number of Successes",
       y = "Probability") +
  theme_minimal()

# Normal plot
ggplot(norm_df, aes(x = x, y = density)) +
  geom_line(color = "darkgreen", linewidth = 1.2) +
  geom_vline(xintercept = mean_salary, color = "red", linetype = "dashed") +
  labs(title = "Normal Distribution",
       x = "Salary",
       y = "Density") +
  theme_minimal()

# Poisson plot
ggplot(pois_df, aes(x = x, y = prob)) +
  geom_col(fill = "orange", color = "black") +
  labs(title = "Poisson Distribution",
       x = "Count",
       y = "Probability") +
  theme_minimal()


# 3. Perform t-test and Chi-Square Test in R
# Load the employee data from 1.R
source("1.R")

# 1. T-test: test whether the average salary differs from 60,000
salary_t_test <- t.test(df$SALARY, mu = 60000)

# 2. Create categories for a chi-square test
# Salary category: High (>= 60,000) or Low (< 60,000)
df$salary_group <- ifelse(df$SALARY >= 60000, "High", "Low")

# Experience category: Experienced if joined before 2020, otherwise New
# (using the STARTDATE column from the data frame)
df$experience_group <- ifelse(df$STARTDATE <= as.Date("2019-12-31"), "Experienced", "New")

# Contingency table
chi_table <- table(df$salary_group, df$experience_group)

# 3. Chi-square test of independence
chi_square_test <- chisq.test(chi_table)

# Print results
cat("=== T-test ===\n")
print(salary_t_test)

cat("\n=== Contingency Table ===\n")
print(chi_table)

cat("\n=== Chi-Square Test ===\n")
print(chi_square_test)


# 4. Calculate Correlation and Build a Simple Linear Regression Model
df <- data.frame(
  Experience = c(1,2,3,4,5,6,7,8,9,10),
  Salary = c(25000,30000,35000,42000,48000,
             52000,58000,65000,72000,80000)
)

print(df)

correlation <- cor(df$Experience,
                   df$Salary)

cat("Correlation =", correlation, "\n")

plot(df$Experience,
     df$Salary,
     main="Experience vs Salary",
     xlab="Experience",
     ylab="Salary",
     pch=19,
     col="blue")

model <- lm(Salary ~ Experience,
            data=df)

summary(model)

coef(model)

newdata <- data.frame(
  Experience=12
)

predict(model,
        newdata)

abline(model,
       col="red",
       lwd=2)


# 5. Conduct Exploratory Data Analysis (EDA) on a Real-World Dataset
data(mtcars)

head(mtcars)

str(mtcars)

dim(mtcars)

names(mtcars)

summary(mtcars)

colSums(is.na(mtcars))

sum(duplicated(mtcars))

cor(mtcars)

hist(mtcars$mpg,
     main="Histogram of MPG",
     xlab="Miles Per Gallon",
     col="lightblue")

boxplot(mtcars$hp,
        main="Horsepower",
        col="orange")

plot(mtcars$wt,
     mtcars$mpg,
     main="Weight vs MPG",
     xlab="Weight",
     ylab="MPG",
     pch=19,
     col="blue")


# 6. Apply Feature Engineering: Scaling, Normalization, and Encoding
df <- data.frame(
  Name = c("Alice","Bob","Charlie","David","Eva"),
  Age = c(23,28,35,40,30),
  Salary = c(30000,45000,60000,80000,55000),
  Gender = c("Female","Male","Male","Male","Female")
)

print(df)

# Scaling
scaled_data <- scale(df[,c("Age","Salary")])

print(scaled_data)

# Normalization
normalize <- function(x){
  (x-min(x))/(max(x)-min(x))
}

normalized_salary <- normalize(df$Salary)

print(normalized_salary)

# Encoding
df$Gender_Encoded <- ifelse(df$Gender=="Male",1,0)

# Add engineered features
df$Age_Scaled <- scaled_data[,1]
df$Salary_Scaled <- scaled_data[,2]
df$Salary_Normalized <- normalized_salary

print(df)


# 7. Practice R Programming: Variables, Control Structures, and Functions
# Variables
num1 <- 20
num2 <- 10
name <- "Alice"

cat("Name:", name, "\n")
cat("Number 1:", num1, "\n")
cat("Number 2:", num2, "\n\n")

# Arithmetic Operators
cat("Addition:", num1 + num2, "\n")
cat("Subtraction:", num1 - num2, "\n")
cat("Multiplication:", num1 * num2, "\n")
cat("Division:", num1 / num2, "\n")
cat("Modulus:", num1 %% num2, "\n")
cat("Power:", num1 ^ 2, "\n\n")

# Relational Operators
cat("num1 > num2 :", num1 > num2, "\n")
cat("num1 == num2 :", num1 == num2, "\n\n")

# If...Else Statement
if(num1 > num2){
  cat("num1 is greater than num2\n\n")
}else{
  cat("num2 is greater than or equal to num1\n\n")
}

# For Loop
cat("For Loop:\n")
for(i in 1:5){
  cat(i, "\n")
}

cat("\n")

# While Loop
cat("While Loop:\n")
count <- 1

while(count <= 5){
  cat(count, "\n")
  count <- count + 1
}

cat("\n")

# User-defined Function
square <- function(x){
  return(x * x)
}

result <- square(8)

cat("Square of 8 =", result)


# 8. Read and Write Data from CSV, Excel, JSON, and XML Files
library(readxl)
library(writexl)
library(jsonlite)
library(XML)

df <- data.frame(
  ID = c(101,102,103,104),
  Name = c("Alice","Bob","Charlie","David"),
  Age = c(22,24,23,25),
  Salary = c(35000,42000,39000,45000)
)

print(df)

# CSV
write.csv(df, "employees.csv", row.names = FALSE)

csv_data <- read.csv("employees.csv")
print(csv_data)

# Excel
write_xlsx(df, "employees.xlsx")

excel_data <- read_excel("employees.xlsx")
print(excel_data)

# JSON
write_json(df, "employees.json", pretty = TRUE)

json_data <- read_json("employees.json")
print(json_data)

# XML
xml_data <- newXMLDoc()

root <- newXMLNode("Employees", doc = xml_data)

for(i in 1:nrow(df))
{
  emp <- newXMLNode("Employee", parent = root)

  newXMLNode("ID", df$ID[i], parent = emp)
  newXMLNode("Name", df$Name[i], parent = emp)
  newXMLNode("Age", df$Age[i], parent = emp)
  newXMLNode("Salary", df$Salary[i], parent = emp)
}

saveXML(xml_data, file = "employees.xml")

xml_file <- xmlParse("employees.xml")

print(xmlRoot(xml_file))


# 9. Use dplyr and tidyr for Data Wrangling Tasks
library(dplyr)
library(tidyr)

df <- data.frame(
  ID = c(101,102,103,104,105),
  Name = c("Alice","Bob","Charlie","David","Eva"),
  Department = c("HR","IT","IT","Sales","HR"),
  Salary = c(35000,50000,48000,45000,40000),
  Age = c(25,28,30,27,24)
)

print(df)

# Select columns
select(df, Name, Salary)

# Filter rows
filter(df, Salary > 40000)

# Arrange rows
arrange(df, desc(Salary))

# Create a new column
df2 <- mutate(df, NewSalary = Salary * 1.10)
print(df2)

# Group and summarize
group_by(df, Department) %>%
  summarise(AverageSalary = mean(Salary))

# Convert wide to long
long_df <- pivot_longer(
  df,
  cols = c(Salary, Age),
  names_to = "Attribute",
  values_to = "Value"
)

print(long_df)

# Convert long to wide
wide_df <- pivot_wider(
  long_df,
  names_from = Attribute,
  values_from = Value
)

print(wide_df)


# 10. Handle Missing Data and Detect Outliers
df <- data.frame(
  Name = c("Alice","Bob","Charlie","David","Eva","Frank"),
  Age = c(22,25,NA,30,28,45),
  Salary = c(35000,42000,39000,45000,150000,40000)
)

print(df)

# Check missing values
print(is.na(df))

# Count missing values
print(colSums(is.na(df)))

# Remove rows with missing values
clean_df <- na.omit(df)

cat("\nDataset after removing missing values:\n")
print(clean_df)

# Replace missing values with mean
mean_age <- mean(df$Age, na.rm = TRUE)

df$Age[is.na(df$Age)] <- mean_age

cat("\nDataset after replacing missing values:\n")
print(df)

# Detect outliers
boxplot(df$Salary,
        main = "Salary Boxplot",
        col = "lightblue")

outliers <- boxplot.stats(df$Salary)$out

cat("\nOutliers:\n")
print(outliers)

# 11. Work with Dates and Times in R
library(lubridate)

# Create dates and times
dates <- c("2023-01-15", "2023-06-20", "2023-12-05")
parsed_dates <- ymd(dates)

# Extract components
years <- year(parsed_dates)
months <- month(parsed_dates, label = TRUE)
days <- day(parsed_dates)

# Date arithmetic
date_diff <- parsed_dates[3] - parsed_dates[1]

cat("Parsed Dates:", as.character(parsed_dates), "\n")
cat("Months:", as.character(months), "\n")
cat("Difference in Days:", date_diff, "\n")


# 12. Visualize Data Using ggplot2 (Bar, Scatter, Histogram, Boxplot)
library(ggplot2)

data(mpg)

# Bar Plot
ggplot(mpg, aes(x = class)) +
  geom_bar(fill = "steelblue") +
  theme_minimal() +
  labs(title = "Bar Plot: Car Classes", x = "Class", y = "Count")

# Scatter Plot
ggplot(mpg, aes(x = displ, y = hwy, color = class)) +
  geom_point() +
  theme_minimal() +
  labs(title = "Scatter Plot: Engine Displacement vs Highway MPG")

# Histogram
ggplot(mpg, aes(x = hwy)) +
  geom_histogram(binwidth = 2, fill = "coral", color = "black") +
  theme_minimal() +
  labs(title = "Histogram: Highway MPG", x = "Highway MPG")

# Boxplot
ggplot(mpg, aes(x = class, y = hwy, fill = class)) +
  geom_boxplot() +
  theme_minimal() +
  labs(title = "Boxplot: Highway MPG by Class")


# 13. Perform K-Means Clustering and Visualize Clusters
library(ggplot2)

data(iris)

# Select numerical features
iris_data <- iris[, 1:4]

# Perform K-Means clustering (k = 3)
set.seed(123)
kmeans_res <- kmeans(iris_data, centers = 3, nstart = 20)

# Add cluster labels to dataset
iris$Cluster <- as.factor(kmeans_res$cluster)

# Visualize clusters
ggplot(iris, aes(x = Sepal.Length, y = Sepal.Width, color = Cluster, shape = Cluster)) +
  geom_point(size = 3) +
  theme_minimal() +
  labs(title = "K-Means Clustering on Iris Dataset")


# 14. Evaluate Models Using Confusion Matrix, Accuracy, and ROC Curve
library(caret)
library(pROC)

data(iris)
binary_iris <- iris[iris$Species != "setosa", ]
binary_iris$Species <- droplevels(binary_iris$Species)

set.seed(123)
train_idx <- createDataPartition(binary_iris$Species, p = 0.7, list = FALSE)
train_data <- binary_iris[train_idx, ]
test_data <- binary_iris[-train_idx, ]

model <- glm(Species ~ Sepal.Length + Sepal.Width, data = train_data, family = binomial)
prob <- predict(model, newdata = test_data, type = "response")
pred_class <- factor(ifelse(prob > 0.5, "virginica", "versicolor"), levels = levels(test_data$Species))

# Confusion Matrix & Accuracy
conf_matrix <- confusionMatrix(pred_class, test_data$Species)
print(conf_matrix)

# ROC Curve
roc_curve <- roc(test_data$Species, prob)
plot(roc_curve, col = "blue", main = "ROC Curve")
cat("AUC:", auc(roc_curve), "\n")


# 15. Perform Text Mining and Create a Word Cloud
library(tm)
library(wordcloud)

text_data <- c("R is a great language for data science and analytics.",
               "Data visualization and machine learning with R are powerful.",
               "Text mining in R allows easy analysis of unstructured text data.")

corpus <- Corpus(VectorSource(text_data))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("english"))

dtm <- TermDocumentMatrix(corpus)
matrix <- as.matrix(dtm)
words <- sort(rowSums(matrix), decreasing = TRUE)
df <- data.frame(word = names(words), freq = words)

wordcloud(words = df$word, freq = df$freq, min.freq = 1,
          max.words = 100, random.order = FALSE, colors = brewer.pal(8, "Dark2"))


# 16. Time Series Forecasting with ARIMA on a real dataset
library(forecast)

data("AirPassengers")

# Fit ARIMA model
fit <- auto.arima(AirPassengers)
summary(fit)

# Forecast next 12 months
forecasted <- forecast(fit, h = 12)

# Plot forecast
plot(forecasted, main = "AirPassengers ARIMA Forecast", xlab = "Year", ylab = "Passengers")


# 17. Create interactive bar, line, and scatter plots using plotly
library(plotly)

data(mtcars)

# Interactive Scatter Plot
plot_ly(data = mtcars, x = ~wt, y = ~mpg, type = "scatter", mode = "markers", color = ~as.factor(cyl)) %>%
  layout(title = "Interactive Scatter Plot: Weight vs MPG")

# Interactive Line Plot
df_time <- data.frame(Day = 1:10, Sales = c(10, 15, 13, 18, 20, 25, 22, 30, 28, 35))
plot_ly(data = df_time, x = ~Day, y = ~Sales, type = "scatter", mode = "lines+markers") %>%
  layout(title = "Interactive Line Plot: Daily Sales")

# Interactive Bar Plot
plot_ly(data = mtcars, x = ~as.factor(cyl), type = "histogram") %>%
  layout(title = "Interactive Bar Plot: Cylinder Counts", xaxis = list(title = "Cylinders"))


# 18. Develop a Shiny app that lets users upload a CSV file
library(shiny)

ui <- fluidPage(
  titlePanel("CSV File Uploader"),
  sidebarLayout(
    sidebarPanel(
      fileInput("file1", "Choose CSV File", accept = ".csv"),
      tags$hr(),
      checkboxInput("header", "Header", TRUE)
    ),
    mainPanel(
      tableOutput("contents")
    )
  )
)

server <- function(input, output) {
  output$contents <- renderTable({
    req(input$file1)
    read.csv(input$file1$datapath, header = input$header)
  })
}

shinyApp(ui = ui, server = server)