We propose to contextualize a glossary of terms on a set of short texts sourced from Wikipedia or automatically generated by LLMs. More specifically, given a corpus of texts \(C\) and a small lexicon \(L\) characteristic of a specific theme or type of discourse, we aim to find the most significant associations between the constituent terms of this lexicon. We will base these associations on their direct co-occurrences within the corpus C as well as on different vector representations of these terms: - Lexical Embeddings computed over the textual content of the corpus C, disregarding document boundaries. - Probabilistic Modeling (LDA) of term co-occurrences within the documents of C, without considering word order (i.e., using a bag-of-words approach).

1 Lexicon definition

1.1 Seed words

library(readr)

# lexiques
 cnx=mycnx()
 df=dbGetQuery(cnx,"SELECT * FROM lexique")
 dbDisconnect(cnx)
## [1] TRUE
 termes_ref=df$terms[order(df$terms)]
 write_csv(df,"lexique.csv",col_names = FALSE)
termes_ref
##   [1] "agriculture"       "antisémitisme"     "artisan"          
##   [4] "artisanat"         "autonomie"         "autoritaire"      
##   [7] "autoritarisme"     "autorité"          "banlieue"         
##  [10] "catholicisme"      "catholique"        "chauvin"          
##  [13] "chauvinisme"       "chrétien"          "chrétienne"       
##  [16] "chrétienté"        "christianisme"     "civilisation"     
##  [19] "civique"           "communautaire"     "communautarisme"  
##  [22] "communautariste"   "communauté"        "conspiration"     
##  [25] "conspirationnisme" "conspirationniste" "défense"          
##  [28] "délinquance"       "délinquant"        "démagogie"        
##  [31] "démocratie"        "démocratique"      "drapeau"          
##  [34] "droite"            "écologie"          "économie"         
##  [37] "élite"             "élites"            "énergie"          
##  [40] "ensauvagement"     "etat"              "etat-nation"      
##  [43] "ethnique"          "étranger"          "européenne"       
##  [46] "extérieur"         "extrême"           "extrême-droite"   
##  [49] "extrême-gauche"    "extrêmes"          "extrémisme"       
##  [52] "extrémiste"        "fascisme"          "féminisme"        
##  [55] "français"          "française"         "françaises"       
##  [58] "france"            "francophone"       "frontière"        
##  [61] "frontières"        "gauche"            "gay"              
##  [64] "genre"             "héritage"          "hijab"            
##  [67] "histoire"          "identitaire"       "identité"         
##  [70] "idéologie"         "immigration"       "impérialisme"     
##  [73] "indépendance"      "individualisme"    "individualiste"   
##  [76] "insécurité"        "intérieur"         "international"    
##  [79] "islam"             "islamisme"         "islamiste"        
##  [82] "islamophobie"      "isolationnisme"    "laïcité"          
##  [85] "liberté"           "local"             "locale"           
##  [88] "locales"           "localisme"         "locaux"           
##  [91] "migratoire"        "minorités"         "nation"           
##  [94] "nativisme"         "néonationalisme"   "nucléaire"        
##  [97] "patrie"            "patriote"          "paysan"           
## [100] "pénalisation"      "peuple"            "polarisation"     
## [103] "populisme"         "populiste"         "protection"       
## [106] "puissance"         "québec"            "québécois"        
## [109] "québécoise"        "québécoises"       "racisme"          
## [112] "raciste"           "radical"           "radicalisation"   
## [115] "régionalisme"      "repli"             "rural"            
## [118] "ruralité"          "sécularisme"       "sécuritaire"      
## [121] "sécurité"          "séparatisme"       "souveraineté"     
## [124] "souverainisme"     "souverainiste"     "supranationalisme"
## [127] "tradition"         "union"             "unité"            
## [130] "valeur"            "xénophobe"         "xénophobie"       
## [133] "zone"

1.2 Stop Word List

In a combinatorial approach where articles are treated as bags-of-words and the goal is not to characterize writing style but rather the more or less systematic associations between terms, stop words tend to generate non-informative and computationally expensive association cliques.

We use the French stop word list from the stopwords-iso project, which can be further augmented with terms specific to the domain under consideration.

 swl <- readLines("stopwords-fr.txt")
 myswl=c("aussi","autres","autre","beaucoup", "amp","url","have","faire","tout","parfois","l","d","on","http","n","https","lors","alors","déjà")
 swl=unique(c(swl,myswl))

2 Corpus

We selected the 300 French Wikipedia pages created before 2022-03-15 and after 2003-03-01 that best match the boolean query “politique & (france | français | française)”. For each page, we prompted every tested LLM to generate an equivalent summary using only the page’s title.

2.1 Statistics

library(readr)
DWP <- read_delim("DWP01_pool/DWP_pool_s.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (2): title, text
## dbl  (2): doc_id, score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_G <- read_delim("DWP01_pool/DWP_pool_G.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (2): title, text
## dbl  (2): doc_id, score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_Gmax <- read_delim("DWP01_pool/DWP_pool_Gmax.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (2): title, text
## dbl  (2): doc_id, score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_M <- read_delim("DWP01_pool/DWP_pool_M.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (2): title, text
## dbl  (2): doc_id, score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_Mb <- read_delim("DWP01_pool/DWP_pool_Msmall.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Warning: One or more parsing issues, call `problems()` on your data frame for details,
## e.g.:
##   dat <- vroom(...)
##   problems(dat)
## Rows: 290 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (3): doc_id, title, text
## dbl  (1): score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_L <- read_delim("DWP01_pool/DWP_pool_L.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (2): title, text
## dbl  (2): doc_id, score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
DWP_Q <- read_delim("DWP01_pool/DWP_pool_Q.tsv", delim = "\t", escape_double = FALSE, trim_ws = TRUE)
## Warning: One or more parsing issues, call `problems()` on your data frame for details,
## e.g.:
##   dat <- vroom(...)
##   problems(dat)
## Rows: 300 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: "\t"
## chr  (3): doc_id, title, text
## dbl  (1): score
## dttm (1): date
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
hist(DWP$date,breaks = "months",freq=TRUE,main="Selected wikipedia abstracts",xlab="Dates des pages")

cat("\nWP text lengths :","\n----\n")
## 
## WP text lengths : 
## ----
summary(nchar(DWP$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    59.0   305.0   628.5  1173.2  1216.8 39574.0
cat("Gemma text lengths :","\n----\n")
## Gemma text lengths : 
## ----
summary(nchar(DWP_G$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##  1586.0  2413.5  2654.5  2671.7  2936.8  4081.0
cat("Gemma3:27b text lengths :","\n----\n")
## Gemma3:27b text lengths : 
## ----
summary(nchar(DWP_Gmax$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   765.0  1245.2  1469.0  1503.4  1653.8  2857.0
cat("Mistral text lengths :","\n----\n")
## Mistral text lengths : 
## ----
summary(nchar(DWP_M$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   454.0   776.0  1037.0  1149.5  1413.5  3656.0
cat("Mistral Small text lengths :","\n----\n")
## Mistral Small text lengths : 
## ----
summary(nchar(DWP_Mb$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   382.0   629.5   720.0   821.7   845.5 10174.0       3
cat("Llama text lengths :","\n----\n")
## Llama text lengths : 
## ----
summary(nchar(DWP_L$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   569.0  1052.0  1229.0  1272.9  1454.8  2397.0
cat("Qwen text length :","\n----\n")
## Qwen text length : 
## ----
summary(nchar(DWP_Q$text))
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    3.00  195.50  292.00  344.18  400.00 2774.00       1

2.2 Content

DWPm=data.frame(doc_id=DWP$doc_id,title=DWP$title,ref=DWP$text,Gemma=DWP_G$text,Gemma27b=DWP_Gmax$text,Mistral=DWP_M$text,Llama=DWP_L$text,Qwen=DWP_Q$text)
DWPm[1:30,]

2.3 Content Evaluation

2.3.1 ROUGE1

library(rougeR)
RevalG=data.frame(LLM=c("Gemma") ,rouge_n(DWPm$Gemma,DWPm$ref,n = 1))
RevalG27b=data.frame(LLM=c("Gemma27b") ,rouge_n(DWPm$Gemma27b,DWPm$ref,n = 1))
RevalM=data.frame(LLM=c("Mistral") ,rouge_n(DWPm$Mistral,DWPm$ref,n = 1))
DWPmb=merge.data.frame(DWP,DWP_Mb,by = "doc_id")
RevalMb=data.frame(LLM=c("Mistral Small"),rouge_n(DWPmb$text.y,DWPmb$text.x,n = 1))
RevalL=data.frame(LLM=c("Llama") ,rouge_n(DWPm$Llama,DWPm$ref,n = 1))
RevalQ=data.frame(LLM=c("Qwen") ,rouge_n(DWPm$Qwen,DWPm$ref,n = 1))
Reval=rbind.data.frame(RevalG,RevalG27b,RevalM,RevalMb,RevalL,RevalQ)
rank=order(Reval$f_measure,decreasing = 1)
Reval[rank,]

2.3.2 ROUGE2

library(rougeR)
RevalG=data.frame(LLM=c("Gemma") ,rouge_n(DWPm$Gemma,DWPm$ref,n = 2))
RevalG27b=data.frame(LLM=c("Gemma27b") ,rouge_n(DWPm$Gemma27b,DWPm$ref,n = 2))
RevalM=data.frame(LLM=c("Mistral") ,rouge_n(DWPm$Mistral,DWPm$ref,n = 2))
RevalMb=data.frame(LLM=c("Mistral Small"),rouge_n(DWPmb$text.y,DWPmb$text.x,n = 2))
RevalL=data.frame(LLM=c("Llama") ,rouge_n(DWPm$Llama,DWPm$ref,n = 2))
RevalQ=data.frame(LLM=c("Qwen") ,rouge_n(DWPm$Qwen,DWPm$ref,n = 2))
Reval=rbind.data.frame(RevalG,RevalG27b,RevalM,RevalMb,RevalL,RevalQ)
rank=order(Reval$f_measure,decreasing = 1)
Reval[rank,]

2.3.3 ROUGE3

library(rougeR)
RevalG=data.frame(LLM=c("Gemma") ,rouge_n(DWPm$Gemma,DWPm$ref,n = 3))
RevalG27b=data.frame(LLM=c("Gemma27b") ,rouge_n(DWPm$Gemma27b,DWPm$ref,n = 3))
RevalM=data.frame(LLM=c("Mistral") ,rouge_n(DWPm$Mistral,DWPm$ref,n = 3))
RevalMb=data.frame(LLM=c("Mistral Small"),rouge_n(DWPmb$text.y,DWPmb$text.x,n = 3))
RevalL=data.frame(LLM=c("Llama") ,rouge_n(DWPm$Llama,DWPm$ref,n = 3))
RevalQ=data.frame(LLM=c("Qwen") ,rouge_n(DWPm$Qwen,DWPm$ref,n = 3))
Reval=rbind.data.frame(RevalG,RevalG27b,RevalM,RevalMb,RevalL,RevalQ)
rank=order(Reval$f_measure,decreasing = 1)
Reval[rank,]

2.4 Correlation of Lexicon Terms within this Corpus

We can investigate the co-occurrence context of the lexicon terms within this corpus. We use Kendall’s test to evaluate the significance of these co-occurrences.

w1="républi"
w2="gauche"
mycontext(D = DWP,w1,w2)
mycontext(D = DWP_G,w1,w2)
mycontext(D = DWP_M,w1,w2)
mycontext(D = DWP_L,w1,w2)
mycontext(D = DWP_Q,w1)
w1="républi"
w2="droite"
mycontext(D = DWP,w1,w2)
mycontext(D = DWP_G,w1,w2)
mycontext(D = DWP_M,w1,w2)
mycontext(D = DWP_L,w1,w2)
mycontext(D = DWP_Q,w1)

3 Representation of Lexicon Terms based on their Collocations (Lexical Embeddings)

A new vector representation is computed solely on the extracted context. This representation aims to predict word collocations across the entire text.

3.1 Word Embeddings

options(digits=8)
library(word2vec)
wem_WP=wempol(DWP,k=20,i = 100, slw = swl)
we_WP=t(as.matrix(wem_WP))
wem_WP_G=wempol(DWP_G,k=20,i = 100, slw = swl)
we_WP_G=t(as.matrix(wem_WP_G))
wem_WP_G27b=wempol(DWP_Gmax,k=20,i = 100, slw = swl)
we_WP_G27b=t(as.matrix(wem_WP_G27b))
wem_WP_M=wempol(DWP_M,k=20,i = 100, slw = swl)
we_WP_M=t(as.matrix(wem_WP_M))
wem_WP_L=wempol(DWP_L,k=20,i = 100, slw = swl)
we_WP_L=t(as.matrix(wem_WP_L))
wem_WP_Q=wempol(DWP_Q,k=20,i = 100, slw = swl)
we_WP_Q=t(as.matrix(wem_WP_Q))

3.2 Lexicon mapping and visualisation

3.2.1 WikiPedia abstracts

library(stringr)
termes_ref_we_cor_WP=ldabicor(we_WP,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP)
cor_WP_we_idx=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP,termes_ref[cor_WP_we_idx])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 24 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

3.2.2 Gemma abstracts

termes_ref_we_cor_WP_G=ldabicor(we_WP_G,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP_G)
cor_WP_we_idx_G=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP_G[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP_G,termes_ref[cor_WP_we_idx_G])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 28 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

3.2.3 Gemma27b abstracts

termes_ref_we_cor_WP_G27b=ldabicor(we_WP_G27b,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP_G27b)
cor_WP_we_idx_G27b=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP_G27b[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP_G27b,termes_ref[cor_WP_we_idx_G27b])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 31 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

3.2.4 Mistral abstracts

termes_ref_we_cor_WP_M=ldabicor(we_WP_M,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP_M)
cor_WP_we_idx_M=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP_M[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP_M,termes_ref[cor_WP_we_idx_M])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 27 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

3.2.5 Llama abstracts

termes_ref_we_cor_WP_L=ldabicor(we_WP_L,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP_L)
cor_WP_we_idx_L=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP_L[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP_L,termes_ref[cor_WP_we_idx_L])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 24 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

3.2.6 Qwen abstracts

termes_ref_we_cor_WP_Q=ldabicor(we_WP_Q,termes_ref,p=0.01)
as.data.frame(termes_ref_we_cor_WP_Q)
cor_WP_we_idx_Q=termes_ref %in% unique(str_split(paste(names(termes_ref_we_cor_WP_Q[1:25]),collapse = " "),pattern = " "))[[1]]
termviz(we_WP_Q,termes_ref[cor_WP_we_idx_Q])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 8 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4 Multinomial Representation based on Occurrences in Texts (Latent Dirichlet Allocation)

We aim to generate vector representations of the words that allow us to study their indirect co-occurrences (i.e., the use of these words in similar contexts). To achieve this, we account for their frequency of appearance in the texts. Two main approaches exist: factor analysis and probabilistic modeling. Here, we follow the probabilistic approach.

4.1 Text Preprocessing for Ensemble Representations

We address the study of frequently associated sets of terms within this corpus. We disregard the ordering of words within a text to study their associations at the document level.

To achieve this, it is necessary to preprocess the text. We remove words that appear in the previously mentioned stop word list (swl) and eliminate those with very low frequency. Lemmatization could also be performed, but it significantly slows down the process (especially since contextual lemmatization requires a syntactic analysis that accLLM/DWPounts for context).

m_ref=5
pDWP <- mypre(DWP,swl = swl,m = m_ref,lemmatize = 0)
pDWP_G <- mypre(DWP_G,swl = swl,m = m_ref,lemmatize = 0)
pDWP_G27b <- mypre(DWP_Gmax,swl = swl,m = m_ref,lemmatize = 0)
pDWP_M <- mypre(DWP_M,swl = swl,m = m_ref,lemmatize = 0)
pDWP_L <- mypre(DWP_L,swl = swl,m = m_ref,lemmatize = 0)
pDWP_Q <- mypre(DWP_Q,swl = swl,m = m_ref,lemmatize = 0)

4.2 Generation of the Document-Term Matrix

The function mydtm generates the representation matrix where documents are modeled as sets of weighted words. Calling this function requires the following parameters: - A variable name to store the generated matrix. - A vector of preprocessed texts (after stop word removal and optional lemmatization). - A vector containing the identifiers of the texts. LLM/DWP Optionally, a minimum frequency threshold m for the words can be specified (the default is 5).

DTM_WP<-mydtm(pDWP$text,pDWP$doc_id,m = m_ref)
cat("WP vocabulary:",DTM_WP$ncol,"\n")
## WP vocabulary: 735
DTM_WP_G<-mydtm(pDWP_G$text,pDWP_G$doc_id,m = m_ref)
cat("Gemma vocabulary:",DTM_WP_G$ncol,"\n")
## Gemma vocabulary: 1778
DTM_WP_G27b<-mydtm(pDWP_G27b$text,pDWP_G27b$doc_id,m = m_ref)
cat("Gemma27b vocabulary:",DTM_WP_G27b$ncol,"\n")
## Gemma27b vocabulary: 1214
DTM_WP_M<-mydtm(pDWP_M$text,pDWP_M$doc_id,m = m_ref)
cat("Mistral vocabulary:",DTM_WP_M$ncol,"\n")
## Mistral vocabulary: 874
DTM_WP_L<-mydtm(pDWP_L$text,pDWP_L$doc_id,m = m_ref)
cat("Llama vocabulary:",DTM_WP_L$ncol,"\n")
## Llama vocabulary: 969
DTM_WP_Q<-mydtm(pDWP_Q$text,pDWP_Q$doc_id,m = m_ref)
cat("Qwen vocabulary:",DTM_WP_Q$ncol,"\n")
## Qwen vocabulary: 201

4.3 Estimation of the Number of Dimensions

The probabilistic approach is a generative approach. It consists of searching for k word distributions that can explain the co-occurrence phenomena observed in the texts. This process assumes that writing a text involves a prior selection of topics or themes, and that these themes induce different probabilities of appearance for the words. - The model is computed using the function ldapol. The required parameters are: - The name to be assigned to the generated model. - The name of the document-term matrix. - The number of dimensions, k, for the calculated model. - A sampling parameter that determines the number of random draws (iterations).

Determining the number of model dimensions (k) is the most delicate point. Similar to k-means methods, this involves a priori determining the number of “themes” contained within the texts. This choice can be based on various quality measures of the resulting model, which is what the following function executes.

Warning: Running this function can be very time-consuming as it involves calculating and comparing a large number of models.

findk(DTM = DTM_WP)

findk(DTM = DTM_WP_G)

findk(DTM = DTM_WP_G27b)

findk(DTM = DTM_WP_M)

findk(DTM = DTM_WP_L)

findk(DTM = DTM_WP_Q)

Here, we choose to set the number of dimensions (k) as the intersection point between two quality measures.

4.4 Calculation of the LDA Model

Once the number of dimensions (\(k\)) has been selected, we proceed to calculate the model using a high number of draws (iterations) and resampling.

options(digits=8)
lm_WP=ldapol(DTM_WP,k=20,b=300)
lm_WP_G=ldapol(DTM_WP_G,k=20,b=300)
lm_WP_G27b=ldapol(DTM_WP_G27b,k=20,b=300)
lm_WP_M=ldapol(DTM_WP_M,k=20,b=300)
lm_WP_L=ldapol(DTM_WP_L,k=20,b=300)
lm_WP_Q=ldapol(DTM_WP_Q,k=20,b=300)

4.4.1 Topic visulaisation

Based on this probabilistic model, we can also calculate the terms that contribute the most to each dimension. It may then become necessary to augment the stop word list.

4.4.1.1 Wikipedia

df_top=my_top_words(lm_WP)
nm_df_top=c()
T=list()
i=0
for(t in df_top){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top)=nm_df_top
df_top

4.4.1.2 Gemma

df_top_G=my_top_words(lm_WP_G)
nm_df_top=c()
T=list()
i=0
for(t in df_top_G){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top_G)=nm_df_top
df_top_G

4.4.1.3 Gemma27b

df_top_G27b=my_top_words(lm_WP_G27b)
nm_df_top=c()
T=list()
i=0
for(t in df_top_G27b){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top_G27b)=nm_df_top
df_top_G27b

4.4.1.4 Mistral

df_top_M=my_top_words(lm_WP_M)
nm_df_top=c()
T=list()
i=0
for(t in df_top_M){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top_M)=nm_df_top
df_top_M

4.4.1.5 Llama

df_top_L=my_top_words(lm_WP_L)
nm_df_top=c()
T=list()
i=0
for(t in df_top_L){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top_L)=nm_df_top
df_top_L

4.4.1.6 Qwen

df_top_Q=my_top_words(lm_WP_Q)
nm_df_top=c()
T=list()
i=0
for(t in df_top_Q){
  i=i+1
  T[[i]] = t[t %in% termes_ref]
  nm_df_top[i]=paste0(i,":",length(T[[i]]),":",paste0(T[[i]],collapse = " "))
}
names(df_top_Q)=nm_df_top
df_top_Q

4.4.2 Lexicon correlations visualisation

4.4.2.1 WikiPedia

termes_ref_lm_cor_WP=ldabicor(lm_WP,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP)
cor_WP_lm_idx=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP,termes_ref[cor_WP_lm_idx])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 27 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.2.2 Gemma

termes_ref_lm_cor_WP_G=ldabicor(lm_WP_G,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP_G)
cor_WP_lm_idx_G=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP_G[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP_G,termes_ref[cor_WP_lm_idx_G])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 26 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.2.3 Gemma27b

termes_ref_lm_cor_WP_G27b=ldabicor(lm_WP_G27b,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP_G27b)
cor_WP_lm_idx_G27b=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP_G27b[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP_G27b,termes_ref[cor_WP_lm_idx_G27b])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 37 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.2.4 Mistral

termes_ref_lm_cor_WP_M=ldabicor(lm_WP_M,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP_M)
cor_WP_lm_idx_M=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP_M[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP_M,termes_ref[cor_WP_lm_idx_M])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 26 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.2.5 Llama

termes_ref_lm_cor_WP_L=ldabicor(lm_WP_L,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP_L)
cor_WP_lm_idx_L=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP_L[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP_L,termes_ref[cor_WP_lm_idx_L])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 34 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.2.6 Qwen

termes_ref_lm_cor_WP_Q=ldabicor(lm_WP_Q,termes_ref,p=0.01)
as.data.frame(termes_ref_lm_cor_WP_Q)
cor_WP_lm_idx_Q=termes_ref %in% unique(str_split(paste(names(termes_ref_lm_cor_WP_M[1:50]),collapse = " "),pattern = " "))[[1]]
termviz(lm_WP_Q,termes_ref[cor_WP_lm_idx_Q])

## **Results for the Principal Component Analysis (PCA)**
## The analysis was performed on 20 individuals, described by 7 variables
## *The results are available in the following objects:
## 
##    name               description                          
## 1  "$eig"             "eigenvalues"                        
## 2  "$var"             "results for the variables"          
## 3  "$var$coord"       "coord. for the variables"           
## 4  "$var$cor"         "correlations variables - dimensions"
## 5  "$var$cos2"        "cos2 for the variables"             
## 6  "$var$contrib"     "contributions of the variables"     
## 7  "$ind"             "results for the individuals"        
## 8  "$ind$coord"       "coord. for the individuals"         
## 9  "$ind$cos2"        "cos2 for the individuals"           
## 10 "$ind$contrib"     "contributions of the individuals"   
## 11 "$call"            "summary statistics"                 
## 12 "$call$centre"     "mean of the variables"              
## 13 "$call$ecart.type" "standard error of the variables"    
## 14 "$call$row.w"      "weights for the individuals"        
## 15 "$call$col.w"      "weights for the variables"

4.4.3 Evaluation

4.4.3.1 Gemma

lex_cor_W=data.frame(terms=names(termes_ref_lm_cor_WP),termes_ref_lm_cor_WP)
lex_cor_G=data.frame(terms=names(termes_ref_lm_cor_WP_G),termes_ref_lm_cor_WP_G)
lex_cor_WG=merge.data.frame(lex_cor_W,lex_cor_G,by.x = "terms")
lex_cor_WG
lex_cor_name=c("Gemma")
lex_cor_inter=c(length(lex_cor_WG[[1]]))
lex_cor_score=c(lex_cor_WG[[2]]%*%lex_cor_WG[[3]])

Context from reference and LLM.

search_context<-function(DWP,DWPL,lex_cor,p1=0.6,p2=0.8){
  lex_cor=lex_cor_WG
  m=matrix(nrow = 0, ncol = 6)
  colnames(m)=c("terms",names(DWP))
  dfCW=df <- as.data.frame(m)
  dfCL=df <- as.data.frame(m)
  corterms=lex_cor$terms
  for(i in c(1:length(lex_cor$terms))){
    c=lex_cor$terms[i]
    dfcf=data.frame(terms=c)
#  cat(c,"\n")
    xy=strsplit(c," ")[[1]]
    if(lex_cor[[2]][[1]]>p1){
      dfCWi=tryCatch(
        cbind.data.frame(dfcf,mycontext(D=DWP,xy[1],xy[2])),
        error= function(e){
          as.data.frame(m)
        }
      )
      dfCW=rbind.data.frame(dfCW,dfCWi)
    }
#    lex_cor[[2]][[1]]
    if(lex_cor[[3]][[1]]>p2){
      dfCLi=tryCatch(
        cbind.data.frame(dfcf,mycontext(D=DWPL,xy[1],xy[2])),
        error= function(e){
          as.data.frame(m)
        }
      )
      dfCL=rbind.data.frame(dfCL,dfCLi)
    }
  }
  return(list(dfCW,dfCL))
}
#lex_cor_WG
context=search_context(DWP,DWP_G,lex_cor_WG)

context[[1]]
context[[2]]

4.4.3.2 Gemma27b

lex_cor_G27b=data.frame(terms=names(termes_ref_lm_cor_WP_G27b),termes_ref_lm_cor_WP_G27b)
lex_cor_WG27b=merge.data.frame(lex_cor_W,lex_cor_G27b,by.x = "terms")
lex_cor_WG27b
lex_cor_name=c(lex_cor_name,"Gemma27b")
lex_cor_inter=c(lex_cor_inter,length(lex_cor_WG27b[[1]]))
lex_cor_score=c(lex_cor_score,lex_cor_WG27b[[2]]%*%lex_cor_WG27b[[3]])

Context from reference and LLM.

context=search_context(DWP,DWP_Gmax,lex_cor_WG27b,p1=0.6)

context[[1]]
context[[2]]

4.4.3.3 Mistral

lex_cor_M=data.frame(terms=names(termes_ref_lm_cor_WP_M),termes_ref_lm_cor_WP_M)
lex_cor_WM=merge.data.frame(lex_cor_W,lex_cor_M,by.x = "terms")
lex_cor_WM
lex_cor_name=c(lex_cor_name,"Mistral")
lex_cor_inter=c(lex_cor_inter,length(lex_cor_WM[[1]]))
lex_cor_score=c(lex_cor_score,lex_cor_WM[[2]]%*%lex_cor_WM[[3]])

Context from reference and LLM.

context=search_context(DWP,DWP_M,lex_cor_WM,p1=0.6)

context[[1]]
context[[2]]

4.4.3.4 Llama

lex_cor_L=data.frame(terms=names(termes_ref_lm_cor_WP_L),termes_ref_lm_cor_WP_L)
lex_cor_WL=merge.data.frame(lex_cor_W,lex_cor_L,by.x = "terms")
lex_cor_WL
lex_cor_name=c(lex_cor_name,"Llama")
lex_cor_inter=c(lex_cor_inter,length(lex_cor_WL[[1]]))
lex_cor_score=c(lex_cor_score,lex_cor_WL[[2]]%*%lex_cor_WL[[3]])

Context from reference and LLM.

context=search_context(DWP,DWP_L,lex_cor_WL,p1=0.6)

context[[1]]
context[[2]]

4.4.3.5 Qwen

lex_cor_Q=data.frame(terms=names(termes_ref_lm_cor_WP_Q),termes_ref_lm_cor_WP_Q)
lex_cor_WQ=merge.data.frame(lex_cor_W,lex_cor_Q,by.x = "terms")
lex_cor_WQ
lex_cor_name=c(lex_cor_name,"Qwen")
lex_cor_inter=c(lex_cor_inter,length(lex_cor_WQ[[1]]))
lex_cor_score=c(lex_cor_score,lex_cor_WQ[[2]]%*%lex_cor_WQ[[3]])

Context from reference and LLM.

context=search_context(DWP,DWP_L,lex_cor_WQ,p1=0.6)

context[[1]]
context[[2]]

4.4.3.6 Summary

idx_score=order(lex_cor_score,decreasing = 1)
data.frame(name=lex_cor_name,intersect=lex_cor_inter,score=lex_cor_score)[idx_score,]

4.4.4 Searching for Correlated Terms Outside the Lexicon

We use this probabilistic model to search for correlated terms that are outside the study lexicon. This approach is comparable to searching for collocations based on lexical embeddings, except that it is applied at the document level.

x="républ"
y="nation"
z="europe"

4.4.4.1 Wikipedia

knn_lm_WP_x=ldanncor(lm_WP,x)
as.data.frame(knn_lm_WP_x)
kknn_lm_WP_y=ldanncor(lm_WP,y)
as.data.frame(kknn_lm_WP_y)
knn_lm_WP_z=ldanncor(lm_WP,z)
as.data.frame(knn_lm_WP_z)

4.4.4.2 Gemma

knn_lm_WP_x_G=ldanncor(lm_WP_G,x)
as.data.frame(knn_lm_WP_x_G)
kknn_lm_WP_y_G=ldanncor(lm_WP_G,y)
as.data.frame(kknn_lm_WP_y_G)
knn_lm_WP_z_G=ldanncor(lm_WP_G,z)
as.data.frame(knn_lm_WP_z_G)

4.4.4.3 Gemma27b

knn_lm_WP_x_G27b=ldanncor(lm_WP_G27b,x)
as.data.frame(knn_lm_WP_x_G27b)
kknn_lm_WP_y_G27b=ldanncor(lm_WP_G27b,y)
as.data.frame(kknn_lm_WP_y_G27b)
knn_lm_WP_z_G27b=ldanncor(lm_WP_G27b,z)
as.data.frame(knn_lm_WP_z_G27b)

4.4.4.4 Mistral

knn_lm_WP_x_M=ldanncor(lm_WP_M,x)
as.data.frame(knn_lm_WP_x_M)
kknn_lm_WP_y_M=ldanncor(lm_WP_M,y)
as.data.frame(kknn_lm_WP_y_M)
knn_lm_WP_z_M=ldanncor(lm_WP_M,z)
as.data.frame(knn_lm_WP_z_M)

4.4.4.5 Llama

knn_lm_WP_x_L=ldanncor(lm_WP_L,x)
as.data.frame(knn_lm_WP_x_L)
kknn_lm_WP_y_L=ldanncor(lm_WP_L,y)
as.data.frame(kknn_lm_WP_y_L)
knn_lm_WP_z_L=ldanncor(lm_WP_L,z)
as.data.frame(knn_lm_WP_z_L)

4.4.4.6 Qwen

knn_lm_WP_x_Q=ldanncor(lm_WP_Q,x)
as.data.frame(knn_lm_WP_x_Q)
#kknn_lm_WP_y_Q=ldanncor(lm_WP_Q,y)
#as.data.frame(kknn_lm_WP_y_Q)
#knn_lm_WP_z_Q=ldanncor(lm_WP_Q,z)
#as.data.frame(knn_lm_WP_z_Q)