Effect of circular features on model training for IR Example

author:

• Michael Hagmann
• Stefan Riezler

date: 30. April 2021

download: https://github.com/StatNLP/empirical_methods

---

knitr::opts_chunk$set(echo = TRUE, 
                      cache.lazy = FALSE,
                      dev = c("svglite", "pdf", "png"),
                      dpi = 300,
                      fig.path = 'figures/',
                      fig.keep = "high")

#added from: https://github.com/yihui/knitr-examples/blob/master/077-wrap-output.Rmd
library(knitr)
hook_output = knit_hooks$get('output')
knit_hooks$set(output = function(x, options) {
  # this hook is used only when the linewidth option is not NULL
  if (!is.null(n <- options$linewidth)) {
    x = knitr:::split_lines(x)
    # any lines wider than n should be wrapped
    if (any(nchar(x) > n)) x = strwrap(x, width = n)
    x = paste(x, collapse = '\n')
  }
  hook_output(x, options)
})

library(tidyverse)
library(magrittr)
library(glue)
library(mgcv)
library(glmnet)
library(parallel)

#When you want to reproduce the examples, you have to run this code chunk in advance!!
powerSet <- function(set) {
  
    apply(X      = rep(list(c(T,F)), length = length(set)) %>% expand.grid(.),
          MARGIN = 1,
          FUN    = function(...) set[...])
}

fold_id <- function(k = 10, id_var) {
  
  id_var = train_data$queryId
  
  nEntities <- length(unique(id_var))
  if (nEntities %% k != 0) warning("Number of id entities is not a multiple of k!")
  
  pi_id_var <- sample(unique(id_var)) # permutate id_var
  map_foldId <- setNames(nm = sample(unique(id_var)), object = (1:nEntities) %% k + 1) 
  
  return(map_foldId[id_var])
}

searchModel <- function(data, response, features, alpha, fold_ids) {
  
  ls_model <- 
    lapply(X   = alpha,
             FUN = function(alpha, ...) cv.glmnet(alpha = alpha, ...),
             #pars for cv.glmnet other than alpha
             x      = data[features] %>% as.matrix(.),
             y      = train_data[[response]], 
             family = "binomial",
             foldid = fold_ids,
             type.measure = "deviance", #alternative choices: "class" or "auc"
             #pars to mclapply
             )#mc.cores = min(length(alpha), 16))
  
  #select alpha with lowest deviance
  alpha_lowest_dev <- 
    sapply(X   = ls_model,
           FUN = function(model) min(deviance(model$glmnet.fit)))
         
  return(ls_model[[which.min(alpha_lowest_dev)]])
  }

myPredict <- function(model, dataF) {
  predict(model, 
          newx = as.matrix(dataF), 
          s    = "lambda.min", 
          type = "response")[,1] 
}

f1_score <- function(score, labels, threshold = .5) {
  #labels: {0,1}: class labels where 1 refers to positive and zero to negative
  #score:  [0,1]: predictive score
  #threshold: (0,1): a single number that is used to split the score in classes
  
  predicted_class <- as.numeric(score > threshold ) 
  confusion_matrix <- table(predicted = predicted_class, true=labels)
  tp <- tryCatch(confusion_matrix["1","1"], error = function(e) return(0))
  fp <- tryCatch(confusion_matrix["1","0"], error = function(e) return(0))
  fn <- tryCatch(confusion_matrix["0","1"], error = function(e) return(0))
  
  #f1 = tp / (tp + (1 / 2) * (fp + fn))
  return( tp / (tp + (1 / 2) * (fp + fn)) )
}

train_data <- 
  readRDS("data/ir_trainset.rds") %>%
  dplyr::mutate(relevance_dicho  = as.numeric(relevance != 0),
                cited_inventor_num = as.numeric(cited_inventor) - 1,
                cited_examiner_num = as.numeric(cited_examiner) - 1,
                cited_family_num   = as.numeric(cited_family) - 1)


fold_ids = fold_id(id_var = train_data$queryId)

features_base       <- c("score_tfIdf", "score_neural")
features_addCit_i   <- c(features_base, "cited_inventor_num")
features_addCit_ie  <- c(features_addCit_i, "cited_examiner_num")
features_addCit_ief <- c(features_addCit_ie, "cited_family_num")

base_model <- 
  searchModel(data = train_data, 
              response = "relevance_dicho",
              features = features_base,
              fold_ids = fold_ids,
              alpha = seq(from = 0, to = 1,  by = .1))


features_addCit_i <- c(features_base , "cited_inventor_num")
model_addCit_i  <- 
  searchModel(data = train_data, 
            response = "relevance_dicho",
            features = features_addCit_i,
            fold_ids = fold_ids,
            alpha = seq(from = 0, to = 1,  by = .1))

model_addCit_ie  <- 
  searchModel(data = train_data, 
            response = "relevance_dicho",
            features = features_addCit_ie,
            fold_ids = fold_ids,
            alpha = seq(from = 0, to = 1,  by = .1))

model_addCit_ief  <- 
  searchModel(data = train_data, 
            response = "relevance_dicho",
            features = features_addCit_ief,
            fold_ids = fold_ids,
            alpha = seq(from = 0, to = 1,  by = .1))

F1 on training data

f1_train <- 
  train_data %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.))) %>%
  select(queryId, documentId, relevance, relevance_dicho, starts_with("score_b")) %>%
  summarize(f1_b    = f1_score(score_b, labels = relevance_dicho),
            f1_bi   = f1_score(score_bi, labels = relevance_dicho),
            f1_bie  = f1_score(score_bie, labels = relevance_dicho),
            f1_bief = f1_score(score_bief, labels = relevance_dicho))

print(f1_train)

# A tibble: 1 x 4
   f1_b f1_bi f1_bie f1_bief
  <dbl> <dbl>  <dbl>   <dbl>
1 0.738 0.796  0.988       1

F1 on test data

test_data <- 
  readRDS("data/ir_testset.rds") %>%
    mutate(relevance_dicho  = as.numeric(relevance != 0),
           cited_inventor_num = as.numeric(cited_inventor) - 1,
           cited_examiner_num = as.numeric(cited_examiner) - 1,
           cited_family_num   = as.numeric(cited_family) - 1) %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.)))

f1_test <- 
  test_data %>%
  select(queryId, documentId, relevance, relevance_dicho, starts_with("score_b")) %>%
  summarize(f1_b    = f1_score(score_b, labels = relevance_dicho),
            f1_bi   = f1_score(score_bi, labels = relevance_dicho),
            f1_bie  = f1_score(score_bie, labels = relevance_dicho),
            f1_bief = f1_score(score_bief, labels = relevance_dicho))

print(f1_test)

# A tibble: 1 x 4
   f1_b f1_bi f1_bie f1_bief
  <dbl> <dbl>  <dbl>   <dbl>
1 0.728 0.810  0.988       1

BASELINE vs EXTENDED Model (students: all features)

Baseline

student_b_all <- 
  gam(score_b ~ s(score_tfIdf) + s(score_neural) + cited_inventor + cited_examiner + cited_family,
      data = test_data,
      family = binomial)

summary(student_b_all)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

Family: binomial 
Link function: logit 

Formula:
score_b ~ s(score_tfIdf) + s(score_neural) + cited_inventor + 
    cited_examiner + cited_family

Parametric coefficients:
                    Estimate Std. Error z value Pr(>|z|)    
(Intercept)       -5.957e+00  6.200e-02  -96.09   <2e-16 ***
cited_inventoryes -2.478e-15  7.951e-02    0.00        1    
cited_examineryes -2.373e-15  6.330e-02    0.00        1    
cited_familyyes    1.096e-14  2.997e-01    0.00        1    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                  edf Ref.df Chi.sq p-value    
s(score_tfIdf)  1.009  1.018   2030  <2e-16 ***
s(score_neural) 1.004  1.007   3617  <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

R-sq.(adj) =      1   Deviance explained =  100%
UBRE = -0.99989  Scale est. = 1         n = 106690

Extended Model

student_bief_all <- 
  gam(score_bief ~ s(score_tfIdf) + s(score_neural) + cited_inventor + cited_examiner + cited_family,
      data = test_data,
      family = binomial)

summary(student_bief_all)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

Family: binomial 
Link function: logit 

Formula:
score_bief ~ s(score_tfIdf) + s(score_neural) + cited_inventor + 
    cited_examiner + cited_family

Parametric coefficients:
                  Estimate Std. Error z value Pr(>|z|)    
(Intercept)        -9.1085     0.3074  -29.63   <2e-16 ***
cited_inventoryes  14.9051     0.7320   20.36   <2e-16 ***
cited_examineryes  15.3045     0.6975   21.94   <2e-16 ***
cited_familyyes    13.6278     1.1251   12.11   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                  edf Ref.df Chi.sq p-value
s(score_tfIdf)  1.013  1.026  2.328   0.132
s(score_neural) 1.004  1.007  0.000   1.000

R-sq.(adj) =      1   Deviance explained =  100%
UBRE = -0.99989  Scale est. = 1         n = 106690

pseudoSmoother_bief <- function(x, model = student_bief_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

pseudoSmoother_b <- function(x, model = student_b_all) {
   #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

par(mfrow = c(3,2), mar=c(2,3,2,3), oma = c(1,1,1,1))
    plot.function(pseudoSmoother_b, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  = "solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher without Citation Feature (F1={round(f1_test$f1_b, 3)})"),
      ylab = "Pseudo Feature Shape", 
      xlab = "citation")
    axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
    axis(2, at = 0:3, labels = 0:3)
    text(labels = "not accessible during teacher training",
         col    = "gray",
         x      = mean(c(-.49, 3.49)),
         y      = mean(c(0,1)),
         adj    = .5)
    
   plot.function(pseudoSmoother_bief, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher with Citation Feature (F1={round(f1_test$f1_bief, 3)})"),
      ylab = "", 
      xlab = "citation")
   axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
   axis(2, at = 0:3, labels = 0:3)
   mtext("citation", side = 4, line = 1)
  
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 1,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(0,1),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))

  plot(student_bief_all, 
       trans = plogis,
       shift = coef(student_bief_all)[1],
       se   = FALSE, 
       #rug  = TRUE, 
       select = 1,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(0, 1))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))
  
  ## Allow a second plot on the same graph
  par(new=TRUE)

## Plot the second plot and put axis scale on right
  plot(student_bief_all, 
       trans = plogis,
       shift = coef(student_bief_all)[1],
       se   = FALSE, 
       select = 1,
       col="black",
       lty = "dashed",
       axes=FALSE,
       ylim = c(0, .006),
       xlim = c(0, 1))
  axis(4, ylim = c(0, .006), at = c(0, .006))
  legend(legend = c("same scale (left y-axis)", "magnified (right y-axis)"), 
         lty    = c("solid", "dashed"),
         bty    = "o",
         bg     = "white",
         x      = "topright")
  
  
  
  mtext("tf-Idf", side = 4, line = 1)
   
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 2,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(-18,10),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))


  plot(student_bief_all,
       shift = coef(student_bief_all)[1],
       trans = plogis,
       se   = FALSE, 
       #rug  = TRUE, 
       select = 2,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(-18, 10))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))
  mtext("neural", side = 4, line = 1)
   
par(mfrow = c(1,1), mar = c(5,4,4,2))

Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”
Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”

Nullification

TEACHER: extended model, STUDENT: no citation

student_bief_nocite <- 
  gam(score_bief ~ s(score_tfIdf) + s(score_neural),
      data = test_data,
      family = binomial)

summary(student_bief_nocite)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

Family: binomial 
Link function: logit 

Formula:
score_bief ~ s(score_tfIdf) + s(score_neural)

Parametric coefficients:
            Estimate Std. Error z value Pr(>|z|)    
(Intercept) -5.43916    0.09429  -57.68   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                  edf Ref.df Chi.sq p-value    
s(score_tfIdf)  7.285  7.994   4097  <2e-16 ***
s(score_neural) 7.506  8.143   4765  <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

R-sq.(adj) =   0.66   Deviance explained = 69.2%
UBRE = -0.85651  Scale est. = 1         n = 106690

pseudoSmoother_bief <- function(x, model = student_bief_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

dev_nocite   <- round(summary(student_bief_nocite)$dev.expl * 100)
dev_all <- round(summary(student_bief_all)$dev.expl * 100)

par(mfrow = c(3,2), mar=c(2,3,2,3), oma = c(1,1,1,1))
    plot.function(pseudoSmoother_b, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      type = "n",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Student without Citation Feature (D\U00B2={dev_nocite}%)"),
      ylab = "", 
      xlab = "")
    axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
    axis(2, at = 0:3, labels = 0:3)
    text(labels = "not included in student model",
         x      = mean(c(-.49, 3.49)),
         y      = mean(c(0,1)),
         adj    = .5)
    
   plot.function(pseudoSmoother_bief, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Student with Citation Feature (D\U00B2={dev_all}%)"),
      ylab = "", 
      xlab = "citation")
   axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
   axis(2, at = 0:3, labels = 0:3)
   mtext("citation", side = 4, line = 1)
  
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 1,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(0,1),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))

  plot(student_bief_all, 
       trans = plogis,
       shift = coef(student_bief_all)[1],
       se   = FALSE, 
       #rug  = TRUE, 
       select = 1,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(0, 1))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))
  
  ## Allow a second plot on the same graph
  par(new=TRUE)

## Plot the second plot and put axis scale on right
  plot(student_bief_all, 
       trans = plogis,
       shift = coef(student_bief_all)[1],
       se   = FALSE, 
       select = 1,
       col="black",
       lty = "dashed",
       axes=FALSE,
       ylim = c(0, .006),
       xlim = c(0, 1))
  axis(4, ylim = c(0, .006), at = c(0, .006))
  legend(legend = c("same scale (left y-axis)", "magnified (right y-axis)"), 
         lty    = c("solid", "dashed"),
         bty    = "o",
         bg     = "white",
         x      = "topright")
  
  
  
  mtext("tf-Idf", side = 4, line = 1)
   
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 2,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(-18,10),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))


  plot(student_bief_all,
       shift = coef(student_bief_all)[1],
       trans = plogis,
       se   = FALSE, 
       #rug  = TRUE, 
       select = 2,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(-18, 10))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))
  mtext("neural", side = 4, line = 1)
   
par(mfrow = c(1,1), mar = c(5,4,4,2))

Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”
Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”

Search for Circularity

feature_list <- 
  c("s(score_tfIdf)", "s(score_neural)", 
    "cited_inventor", "cited_examiner", "cited_family")

Baseline system (without circular features)

target <- "score_b"

fitted_models_b <-
  #generate the power set of feature_list, and remove the void set 
  powerSet(feature_list)[-(2^length(feature_list))] %>%
  #build the symbolic model description  
  sapply(., function(...) glue("{target}~{glue_collapse(..., sep='+')}")) %>%
  #fit the models to data, and extract key statistics
  tibble(formula_str = .,
         models      = lapply(X = .,
                              FUN  = function(m_str,...) gam(as.formula(m_str),  family = binomial, ...),
                              data = test_data[c(target, str_replace(feature_list, "(?:s[(])(.*)(?:[)])", "\\1"))]),
         data_fit    = floor(sapply(models, function(m) summary(m)$dev.expl) * 100),
         complexity  = sapply(models, function(...) attr(logLik(...), 'df'))) %>%
  #Sort the models so that we can find models that replicate the data well.
  #For this models data_fit should be approximately 100.
  arrange(desc(data_fit), complexity)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

Top 10

head(fitted_models_b[,-2])

A tibble: 6 × 3

formula_str	data_fit	complexity
<chr>	<dbl>	<dbl>
score_b~s(score_tfIdf)+s(score_neural)	100	3.012458
score_b~s(score_tfIdf)+s(score_neural)+cited_examiner	100	4.012408
score_b~s(score_tfIdf)+s(score_neural)+cited_inventor+cited_examiner	100	5.012369
score_b~s(score_tfIdf)+s(score_neural)+cited_inventor+cited_examiner+cited_family	100	6.012360
score_b~s(score_tfIdf)+s(score_neural)+cited_inventor	99	4.012449
score_b~s(score_tfIdf)+s(score_neural)+cited_family	99	4.012453

Selected model

summary(fitted_models_b$models[[1]])

Family: binomial 
Link function: logit 

Formula:
score_b ~ s(score_tfIdf) + s(score_neural)

Parametric coefficients:
            Estimate Std. Error z value Pr(>|z|)    
(Intercept)  -5.9574     0.0609  -97.83   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                  edf Ref.df Chi.sq p-value    
s(score_tfIdf)  1.009  1.018   2604  <2e-16 ***
s(score_neural) 1.004  1.007   4267  <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

R-sq.(adj) =      1   Deviance explained =  100%
UBRE = -0.99994  Scale est. = 1         n = 106690

BIEF System (with circular features)

target <- "score_bief"

fitted_models_bief <-
  #generate the power set of feature_list, and remove the void set 
  powerSet(feature_list)[-(2^length(feature_list))] %>%
  #build the symbolic model description  
  sapply(., function(...) glue("{target}~{glue_collapse(..., sep='+')}")) %>%
  #fit the models to data, and extract key statistics
  tibble(formula_str = .,
         models      = lapply(X = .,
                              FUN  = function(m_str,...) gam(as.formula(m_str), family = binomial, ...),
                              data = test_data[c(target, str_replace(feature_list, "(?:s[(])(.*)(?:[)])", "\\1"))]),
         data_fit    = floor(sapply(models, function(m) summary(m)$dev.expl) * 100),
         complexity  = sapply(models, function(...) attr(logLik(...), 'df'))) %>%
  #Sort the models so that we can find models that replicate the data well.
  #For this models data_fit should be approximately 100.
  arrange(desc(data_fit), complexity)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

Top 10

head(fitted_models_bief[,-2])

A tibble: 6 × 3

formula_str	data_fit	complexity
<chr>	<dbl>	<dbl>
score_bief~s(score_tfIdf)+cited_inventor+cited_examiner+cited_family	100	5.013042
score_bief~s(score_tfIdf)+s(score_neural)+cited_inventor+cited_examiner+cited_family	100	6.016468
score_bief~cited_inventor+cited_examiner+cited_family	99	4.000000
score_bief~s(score_neural)+cited_inventor+cited_examiner+cited_family	99	5.003346
score_bief~s(score_tfIdf)+s(score_neural)+cited_inventor+cited_examiner	98	11.377042
score_bief~s(score_tfIdf)+cited_inventor+cited_examiner	97	6.617500

Selcted model

summary(fitted_models_bief$models[[1]])

Family: binomial 
Link function: logit 

Formula:
score_bief ~ s(score_tfIdf) + cited_inventor + cited_examiner + 
    cited_family

Parametric coefficients:
                  Estimate Std. Error z value Pr(>|z|)    
(Intercept)        -9.1085     0.3066  -29.71   <2e-16 ***
cited_inventoryes  14.9051     0.6115   24.37   <2e-16 ***
cited_examineryes  15.3045     0.5518   27.73   <2e-16 ***
cited_familyyes    13.6278     1.0133   13.45   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                 edf Ref.df Chi.sq p-value
s(score_tfIdf) 1.013  1.026  2.398   0.126

R-sq.(adj) =      1   Deviance explained =  100%
UBRE = -0.99991  Scale est. = 1         n = 106690

Ablation study 1

Remove citation information (set all values to zero)

f1_ablation_circ <- 
    train_data %>%
  mutate(cited_inventor_num = 0,
         cited_examiner_num = 0,
         cited_family_num   = 0) %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.))) %>%
  select(queryId, documentId, relevance, relevance_dicho, starts_with("score_b")) %>%
  summarize(f1_b    = f1_score(score_b, labels = relevance_dicho),
            f1_bi   = f1_score(score_bi, labels = relevance_dicho),
            f1_bie  = f1_score(score_bie, labels = relevance_dicho),
            f1_bief = f1_score(score_bief, labels = relevance_dicho))

print(f1_ablation_circ)

# A tibble: 1 x 4
   f1_b f1_bi f1_bie f1_bief
  <dbl> <dbl>  <dbl>   <dbl>
1 0.738 0.721  0.427       0

train_data %>%
  mutate(cited_inventor_num = 0,
         cited_examiner_num = 0,
         cited_family_num   = 0) %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.))) %>%
  select(starts_with("score_b")) %>%
  summary(.)

    score_b             score_bi           score_bie        
 Min.   :0.0000000   Min.   :0.0000000   Min.   :0.0000000  
 1st Qu.:0.0003342   1st Qu.:0.0002366   1st Qu.:0.0000096  
 Median :0.0024591   Median :0.0017703   Median :0.0000579  
 Mean   :0.0577150   Mean   :0.0517937   Mean   :0.0177408  
 3rd Qu.:0.0157711   3rd Qu.:0.0115585   3rd Qu.:0.0003128  
 Max.   :1.0000000   Max.   :1.0000000   Max.   :1.0000000  
   score_bief       
 Min.   :0.0001047  
 1st Qu.:0.0001047  
 Median :0.0001050  
 Mean   :0.0001154  
 3rd Qu.:0.0001064  
 Max.   :0.0017877  

Ablation study 2

Remove all non-circular information (set all values to zero)

f1_ablation_noncirc <- 
  train_data %>%
  mutate(score_tfIdf = 0,
         score_neural = 0) %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.))) %>%
  select(queryId, documentId, relevance, relevance_dicho, starts_with("score_b")) %>%
    summarize(f1_b    = f1_score(score_b, labels = relevance_dicho),
            f1_bi   = f1_score(score_bi, labels = relevance_dicho),
            f1_bie  = f1_score(score_bie, labels = relevance_dicho),
            f1_bief = f1_score(score_bief, labels = relevance_dicho))

print(f1_ablation_noncirc)

# A tibble: 1 x 4
   f1_b f1_bi f1_bie f1_bief
  <dbl> <dbl>  <dbl>   <dbl>
1     0 0.380  0.941       1

train_data %>%
  mutate(score_tfIdf = 0,
         score_neural = 0) %>%
    mutate(
      score_b    = myPredict(base_model,       .[features_base]       %>% as.matrix(.)),
      score_bi   = myPredict(model_addCit_i,   .[features_addCit_i]   %>% as.matrix(.)),
      score_bie  = myPredict(model_addCit_ie,  .[features_addCit_ie]  %>% as.matrix(.)),
      score_bief = myPredict(model_addCit_ief, .[features_addCit_ief] %>% as.matrix(.))) %>%
  select(starts_with("score_b")) %>%
  summary(.)

    score_b          score_bi         score_bie          score_bief       
 Min.   :0.1047   Min.   :0.08015   Min.   :0.001856   Min.   :0.0001047  
 1st Qu.:0.1047   1st Qu.:0.08015   1st Qu.:0.001856   1st Qu.:0.0001047  
 Median :0.1047   Median :0.08015   Median :0.001856   Median :0.0001047  
 Mean   :0.1047   Mean   :0.09262   Mean   :0.053074   Mean   :0.0576162  
 3rd Qu.:0.1047   3rd Qu.:0.08015   3rd Qu.:0.001856   3rd Qu.:0.0001047  
 Max.   :0.1047   Max.   :0.99997   Max.   :0.999983   Max.   :0.9978506  

Teacher models

Baseline

coef(base_model, s=base_model$lambda.min)

3 x 1 sparse Matrix of class "dgCMatrix"
                     1
(Intercept)  -2.145952
score_tfIdf  16.643910
score_neural  1.282155

BIEF model (has access to all features)

coef(model_addCit_ief, s=model_addCit_ief$lambda.min)

6 x 1 sparse Matrix of class "dgCMatrix"
                           1
(Intercept)        -9.164106
score_tfIdf         2.894547
score_neural        .       
cited_inventor_num 14.905101
cited_examiner_num 15.304514
cited_family_num   13.627802

BIE Model (patial circular teacher)

student_bie_all <- 
  gam(score_bie ~ s(score_tfIdf) + s(score_neural) + cited_inventor + cited_examiner + cited_family,
      data = test_data,
      family = binomial)

student_bie_nocite <- 
  gam(score_bie ~ s(score_tfIdf) + s(score_neural),
      data = test_data,
      family = binomial)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

pseudoSmoother_bie <- function(x, model = student_bie_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

pseudoSmoother_b <- function(x, model = student_b_all) {
   #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

par(mfrow = c(3,2), mar=c(2,3,2,3), oma = c(1,1,1,1))
    plot.function(pseudoSmoother_b, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  = "solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher without Citation Feature (F1={round(f1_test$f1_b, 3)})"),
      ylab = "Pseudo Feature Shape", 
      xlab = "citation")
    axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
    axis(2, at = 0:3, labels = 0:3)
    text(labels = "not accessible during teacher training",
         col    = "gray",
         x      = mean(c(-.49, 3.49)),
         y      = mean(c(0,1)),
         adj    = .5)
    
   plot.function(pseudoSmoother_bie, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher with Inventor and Examiner Feature (F1={round(f1_test$f1_bie, 3)})"),
      ylab = "", 
      xlab = "citation")
   axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
   axis(2, at = 0:3, labels = 0:3)
   mtext("citation", side = 4, line = 1)
  
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 1,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(0,1),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))

  plot(student_bie_all, 
       trans = plogis,
       shift = coef(student_bie_all)[1],
       se   = FALSE, 
       #rug  = TRUE, 
       select = 1,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(0, 1))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))
  mtext("tf-Idf", side = 4, line = 1)
   
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 2,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(-18,10),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))


  plot(student_bie_all,
       shift = coef(student_bie_all)[1],
       trans = plogis,
       se   = FALSE, 
       #rug  = TRUE, 
       select = 2,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(-18, 10))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))
  mtext("neural", side = 4, line = 1)
   
par(mfrow = c(1,1), mar = c(5,4,4,2))

Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”
Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”

pseudoSmoother_bie <- function(x, model = student_bie_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

dev_nocite   <- round(summary(student_bie_nocite)$dev.expl * 100)
dev_all <- round(summary(student_bie_all)$dev.expl * 100)

par(mfrow = c(3,2), mar=c(2,3,2,3), oma = c(1,1,1,1))
    plot.function(pseudoSmoother_b, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      type = "n",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Student without Citation Feature (D\U00B2={dev_nocite}%)"),
      ylab = "", 
      xlab = "")
    axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
    axis(2, at = 0:3, labels = 0:3)
    text(labels = "not included in student model",
         x      = mean(c(-.49, 3.49)),
         y      = mean(c(0,1)),
         adj    = .5)
    
   plot.function(pseudoSmoother_bie, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Student with Citation Feature (D\U00B2={dev_all}%)"),
      ylab = "", 
      xlab = "citation")
   axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
   axis(2, at = 0:3, labels = 0:3)
   mtext("citation", side = 4, line = 1)
  
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 1,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(0,1),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))

  plot(student_bie_all, 
       trans = plogis,
       shift = coef(student_bie_all)[1],
       se   = FALSE, 
       #rug  = TRUE, 
       select = 1,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(0, 1))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))
  mtext("tf-Idf", side = 4, line = 1)
   
  plot(student_b_all,
       trans = plogis,
       shift = coef(student_b_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 2,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(-18,10),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))


  plot(student_bie_all,
       shift = coef(student_bie_all)[1],
       trans = plogis,
       se   = FALSE, 
       #rug  = TRUE, 
       select = 2,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(-18, 10))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))
  mtext("neural", side = 4, line = 1)
   
par(mfrow = c(1,1), mar = c(5,4,4,2))

Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”
Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”

BI and BIE Model (all patial circular teachers)

student_bi_all <- 
  gam(score_bi ~ s(score_tfIdf) + s(score_neural) + cited_inventor + cited_examiner + cited_family,
      data = test_data,
      family = binomial)

student_bi_nocite <- 
  gam(score_bi ~ s(score_tfIdf) + s(score_neural),
      data = test_data,
      family = binomial)

Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”
Warning message in eval(family$initialize):
“non-integer #successes in a binomial glm!”

pseudoSmoother_bie <- function(x, model = student_bie_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}

pseudoSmoother_bi <- function(x, model = student_bi_all) {
  #calculate estimated group means
  plogis((coef(model)[1] + c(0, coef(model)[-1]))[round(x) + 1])
}


par(mfrow = c(3,2), mar=c(2,3,2,3), oma = c(1,1,1,1))
    plot.function(pseudoSmoother_bi, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher with Inventor Feature (F1={round(f1_test$f1_bi, 3)})"),
      ylab = "", 
      xlab = "citation")
    axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
    
   plot.function(pseudoSmoother_bie, 
      from = -.49, 
      to   = 3.49, 
      n    = 1001, 
      lty  ="solid",
      xlim = c(-.49, 3.49),
      ylim = c(0,1),
      xaxt = "n",
      yaxt = "n",
      main = glue("Teacher with Inventor and Examiner Feature (F1={round(f1_test$f1_bie, 3)})"),
      ylab = "", 
      xlab = "citation")
   axis(1, at = 0:3, labels = c("none", "inventor", "examiner", "family"))
   mtext("citation", side = 4, line = 1)
  
  plot(student_bi_all,
       trans = plogis,
       shift = coef(student_bi_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 1,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(0,1),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))

  plot(student_bie_all, 
       trans = plogis,
       shift = coef(student_bie_all)[1],
       se   = FALSE, 
       #rug  = TRUE, 
       select = 1,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(0, 1))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_tfIdf))
  mtext("tf-Idf", side = 4, line = 1)
   
  plot(student_bi_all,
       trans = plogis,
       shift = coef(student_bi_all)[1],
       se   = FALSE, 
       #rug  = TRUE,
       select = 2,
       main = "",
       ylab = "", 
       xlab = "",
       ylim = c(-.05,1),
       xlim = c(-18,10),
       yaxt = "n")
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))


  plot(student_bie_all,
       shift = coef(student_bie_all)[1],
       trans = plogis,
       se   = FALSE, 
       #rug  = TRUE, 
       select = 2,
       main = "",
       ylab = "", 
       yaxt = "n",
       xlab = "",
       ylim = c(-.05, 1),
       xlim = c(-18, 10))
  axis(2, at = 0:3, labels = 0:3)
  rug(unique(test_data$score_neural))
  mtext("neural", side = 4, line = 1)
   
par(mfrow = c(1,1), mar = c(5,4,4,2))

Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”
Warning message in rug(unique(test_data$score_neural)):
“some values will be clipped”